Fix

Signed-off-by: Tyler Gu <[email protected]>

data/metrics-server.yaml

@@ -0,0 +1,197 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    k8s-app: metrics-server
+  name: metrics-server
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    k8s-app: metrics-server
+    rbac.authorization.k8s.io/aggregate-to-admin: "true"
+    rbac.authorization.k8s.io/aggregate-to-edit: "true"
+    rbac.authorization.k8s.io/aggregate-to-view: "true"
+  name: system:aggregated-metrics-reader
+rules:
+- apiGroups:
+  - metrics.k8s.io
+  resources:
+  - pods
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    k8s-app: metrics-server
+  name: system:metrics-server
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes/metrics
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    k8s-app: metrics-server
+  name: metrics-server-auth-reader
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+- kind: ServiceAccount
+  name: metrics-server
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    k8s-app: metrics-server
+  name: metrics-server:system:auth-delegator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:auth-delegator
+subjects:
+- kind: ServiceAccount
+  name: metrics-server
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    k8s-app: metrics-server
+  name: system:metrics-server
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:metrics-server
+subjects:
+- kind: ServiceAccount
+  name: metrics-server
+  namespace: kube-system
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    k8s-app: metrics-server
+  name: metrics-server
+  namespace: kube-system
+spec:
+  ports:
+  - name: https
+    port: 443
+    protocol: TCP
+    targetPort: https
+  selector:
+    k8s-app: metrics-server
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    k8s-app: metrics-server
+  name: metrics-server
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      k8s-app: metrics-server
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 0
+  template:
+    metadata:
+      labels:
+        k8s-app: metrics-server
+    spec:
+      containers:
+      - args:
+        - --cert-dir=/tmp
+        - --secure-port=4443
+        - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
+        - --kubelet-use-node-status-port
+        - --metric-resolution=15s
+        - --kubelet-insecure-tls
+        image: registry.k8s.io/metrics-server/metrics-server:v0.6.4
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          failureThreshold: 3
+          httpGet:
+            path: /livez
+            port: https
+            scheme: HTTPS
+          periodSeconds: 10
+        name: metrics-server
+        ports:
+        - containerPort: 4443
+          name: https
+          protocol: TCP
+        readinessProbe:
+          failureThreshold: 3
+          httpGet:
+            path: /readyz
+            port: https
+            scheme: HTTPS
+          initialDelaySeconds: 20
+          periodSeconds: 10
+        resources:
+          requests:
+            cpu: 100m
+            memory: 200Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          runAsNonRoot: true
+          runAsUser: 1000
+        volumeMounts:
+        - mountPath: /tmp
+          name: tmp-dir
+      nodeSelector:
+        kubernetes.io/os: linux
+      priorityClassName: system-cluster-critical
+      serviceAccountName: metrics-server
+      volumes:
+      - emptyDir: {}
+        name: tmp-dir
+---
+apiVersion: apiregistration.k8s.io/v1
+kind: APIService
+metadata:
+  labels:
+    k8s-app: metrics-server
+  name: v1beta1.metrics.k8s.io
+spec:
+  group: metrics.k8s.io
+  groupPriorityMinimum: 100
+  insecureSkipTLSVerify: true
+  service:
+    name: metrics-server
+    namespace: kube-system
+  version: v1beta1
+  versionPriority: 100

performance_measurement/measure_performance.py

@@ -1,5 +1,6 @@
 import argparse
 import dataclasses
+from functools import partial
 import glob
 import json
 import logging
@@ -16,19 +17,20 @@
 from rabbitmq_inputs import RabbitMQInputGenerator
 from stats_watch import WatchStats
 from zk_inputs import ZooKeeperInputGenerator
+import kubernetes
 
 sys.path.insert(0, os.path.abspath(
     os.path.join(os.path.dirname(__file__), '..')))
 
-from acto import utils
-from acto.common import kubernetes_client
-from acto.constant import CONST
-from acto.deploy import Deploy
-from acto.kubectl_client.kubectl import KubectlClient
-from acto.kubernetes_engine import kind
-from acto.lib.operator_config import OperatorConfig
-from acto.post_process.post_chain_inputs import ChainInputs
 from acto.utils.preprocess import process_crd
+from acto.post_process.post_chain_inputs import ChainInputs
+from acto.lib.operator_config import OperatorConfig
+from acto.kubernetes_engine import kind
+from acto.kubectl_client.kubectl import KubectlClient
+from acto.deploy import Deploy
+from acto.constant import CONST
+from acto.common import kubernetes_client
+from acto import utils
 
 
 def load_inputs_from_dir(dir: str) -> list:
@@ -42,6 +44,15 @@ def load_inputs_from_dir(dir: str) -> list:
     return inputs
 
 
+def deploy_metrics_server_f(apiclient: kubernetes.client.ApiClient, kubectl_client: KubectlClient):
+    """Deploy metrics server"""
+    logging.info("Deploying metrics server")
+    p = kubectl_client.kubectl(["apply", "-f", "data/metrics-server.yaml"])
+    if p.returncode != 0:
+        logging.error("Failed to deploy metrics server")
+        return False
+
+
 def test_normal(
         workdir: str, input_dir: str, config: OperatorConfig, sts_name_f: Callable[[dict], str],
         ds_name_f: Callable[[dict], str], modes: list):
@@ -50,7 +61,10 @@ def test_normal(
 
     # start the k8s cluster
     kubeconfig = os.path.join(os.path.expanduser("~"), ".kube", "anvil")
+    deploy_metrics_server_f = partial(
+        deploy_metrics_server_f, kubectl_client=KubectlClient(kubeconfig, context_name))
     cluster = kind.Kind(acto_namespace=0,
+                        posthooks=[deploy_metrics_server_f],
                         feature_gates=config.kubernetes_engine.feature_gates)
     cluster.configure_cluster(config.num_nodes, config.kubernetes_version)
     cluster.restart_cluster(name="anvil", kubeconfig=kubeconfig)
@@ -83,7 +97,8 @@ def test_normal(
         watcher = WatchStats("anvil", control_plane_stats_dir)
         pods_watcher = MetricsApiWatcher(control_plane_stats_dir)
         watcher_thread = threading.Thread(target=watcher.start, args=(cluster,))
-        pods_watcher_thread = threading.Thread(target=pods_watcher.start, args=(runner.apiclient,deploy.operator_name))
+        pods_watcher_thread = threading.Thread(
+            target=pods_watcher.start, args=(runner.apiclient, deploy.operator_name))
         watcher_thread.start()
         pods_watcher_thread.start()
 
@@ -115,7 +130,7 @@ def test_normal(
 
         watcher = WatchStats("anvil", control_plane_stats_dir)
         pods_watcher = MetricsApiWatcher(control_plane_stats_dir)
-        
+
         gen = 0
         for workload in workloads:
             cluster.restart_cluster(name="anvil", kubeconfig=kubeconfig)
@@ -127,7 +142,8 @@ def test_normal(
                 logging.info('Not deployed. Try again!')
 
             watcher_thread = threading.Thread(target=watcher.start, args=(cluster,))
-            pods_watcher_thread = threading.Thread(target=pods_watcher.start, args=(runner.apiclient,deploy.operator_name))
+            pods_watcher_thread = threading.Thread(
+                target=pods_watcher.start, args=(runner.apiclient, deploy.operator_name))
             watcher_thread.start()
             pods_watcher_thread.start()