Update k8s manifests for filebeat and metricbeat in dev-tools (#37401)

* updated manifests from deploy * adding filestream id to all filebeat manifests
elastic · Dec 22, 2023 · 2f7ff01 · 2f7ff01
1 parent e44fc14
commit 2f7ff01
Show file tree

Hide file tree

Showing 8 changed files with 668 additions and 607 deletions.
diff --git a/deploy/kubernetes/filebeat-kubernetes.yaml b/deploy/kubernetes/filebeat-kubernetes.yaml
@@ -113,6 +113,7 @@ data:
   filebeat.yml: |-
     filebeat.inputs:
     - type: filestream
+      id: kubernetes-container-logs-${data.kubernetes.pod.name}-${data.kubernetes.container.id}
       paths:
         - /var/log/containers/*.log
       parsers:

diff --git a/deploy/kubernetes/filebeat/filebeat-configmap.yaml b/deploy/kubernetes/filebeat/filebeat-configmap.yaml
@@ -9,6 +9,7 @@ data:
   filebeat.yml: |-
     filebeat.inputs:
     - type: filestream
+      id: kubernetes-container-logs-${data.kubernetes.pod.name}-${data.kubernetes.container.id}
       paths:
         - /var/log/containers/*.log
       parsers:

diff --git a/dev-tools/kubernetes/Tiltfile b/dev-tools/kubernetes/Tiltfile
@@ -137,7 +137,7 @@ def k8s_expose(
 # `beat`: `metricbeat` to test Metricbeat, `filebeat` to test Filebeat
 # `mode`: `debug` to start a remote debugger that you can connect to from your IDE with hot reloading enabled, `run` to just run Metricbeat without a debugger but still with hot reloading enabled
 # `arch`: `amd64` to build go binary for amd64 architecture, `arm64` to build go binary for arm64 (aka M1 Apple chip) architecture
-# `k8s_env`: `kind` to run against a Kind cluster with no docker repo, `gcp` to use a docker repo on GCP
+# `k8s_env`: `kind` to run against a Kind cluster with no docker repo, `gcp` to use a docker repo on GCP, `aws` to use a docker repo on AWS
 # `k8s_cluster`: `single` to use a single node k8s cluster, `multi` to use a k8s with more than 1 node.
 #       if running on a multi-node cluster we expect to have at least 2 workers and a control plane node. One of the workers (eg. worker1)
 #       should have a taint and a label (for node affinity) to make sure that only the debugger runs on that node. You need to run the following commands:
@@ -203,10 +203,20 @@ def beat(
     k8s_expose(beat=beat, mode=mode, k8s_cluster=k8s_cluster)
 
 
+# Note: Select only one of the following examples or modify one with the parameters you want to use
+
+# Run metricbeat in run mode against a single node k8s cluster with a docker repo on AWS
+# beat(beat="metricbeat",
+#      mode="run",
+#      arch="amd64",
+#      k8s_env="aws",
+#      k8s_cluster="single",
+#      )
+
+# Run on Mac M1 against a single node k8s cluster
 beat(beat="metricbeat",
-    #  mode="debug",
-     mode="run",
-     arch="amd64",
-     k8s_env="aws",
+     mode="debug",
+     arch="arm64",
+     k8s_env="kind",
      k8s_cluster="single",
      )
diff --git a/dev-tools/kubernetes/filebeat/manifest.debug.yaml b/dev-tools/kubernetes/filebeat/manifest.debug.yaml
@@ -1,3 +1,106 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: filebeat
+  namespace: kube-system
+  labels:
+    k8s-app: filebeat
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: filebeat
+  labels:
+    k8s-app: filebeat
+rules:
+- apiGroups: [""] # "" indicates the core API group
+  resources:
+  - namespaces
+  - pods
+  - nodes
+  verbs:
+  - get
+  - watch
+  - list
+- apiGroups: ["apps"]
+  resources:
+    - replicasets
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["batch"]
+  resources:
+    - jobs
+  verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: filebeat
+  # should be the namespace where filebeat is running
+  namespace: kube-system
+  labels:
+    k8s-app: filebeat
+rules:
+  - apiGroups:
+      - coordination.k8s.io
+    resources:
+      - leases
+    verbs: ["get", "create", "update"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: filebeat-kubeadm-config
+  namespace: kube-system
+  labels:
+    k8s-app: filebeat
+rules:
+  - apiGroups: [""]
+    resources:
+      - configmaps
+    resourceNames:
+      - kubeadm-config
+    verbs: ["get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: filebeat
+subjects:
+- kind: ServiceAccount
+  name: filebeat
+  namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: filebeat
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: filebeat
+  namespace: kube-system
+subjects:
+  - kind: ServiceAccount
+    name: filebeat
+    namespace: kube-system
+roleRef:
+  kind: Role
+  name: filebeat
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: filebeat-kubeadm-config
+  namespace: kube-system
+subjects:
+  - kind: ServiceAccount
+    name: filebeat
+    namespace: kube-system
+roleRef:
+  kind: Role
+  name: filebeat-kubeadm-config
+  apiGroup: rbac.authorization.k8s.io
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -9,9 +112,17 @@ metadata:
 data:
   filebeat.yml: |-
     filebeat.inputs:
-    - type: container
+    - type: filestream
+      id: kubernetes-container-logs-${data.kubernetes.pod.name}-${data.kubernetes.container.id}
       paths:
         - /var/log/containers/*.log
+      parsers:
+        - container: ~
+      prospector:
+        scanner:
+          fingerprint.enabled: true
+          symlinks: true
+      file_identity.fingerprint: ~
       processors:
         - add_kubernetes_metadata:
             host: ${NODE_NAME}
@@ -20,15 +131,23 @@ data:
                 logs_path: "/var/log/containers/"
 
     # To enable hints based autodiscover, remove `filebeat.inputs` configuration and uncomment this:
-    #filebeat.autodiscover:
+    # filebeat.autodiscover:
     #  providers:
     #    - type: kubernetes
     #      node: ${NODE_NAME}
     #      hints.enabled: true
     #      hints.default_config:
-    #        type: container
+    #        type: filestream
+    #        id: kubernetes-container-logs-${data.kubernetes.pod.name}-${data.kubernetes.container.id}
     #        paths:
-    #          - /var/log/containers/*${data.kubernetes.container.id}.log
+    #        - /var/log/containers/*-${data.kubernetes.container.id}.log
+    #        parsers:
+    #        - container: ~
+    #        prospector:
+    #         scanner:
+    #           fingerprint.enabled: true
+    #           symlinks: true
+    #        file_identity.fingerprint: ~
 
     processors:
       - add_cloud_metadata:
@@ -71,7 +190,6 @@ spec:
         args: [
           "-c", "/etc/filebeat.yml",
           "-e",
-          "-system.hostfs=/hostfs",
         ]
         ports:
         - containerPort: 56268
@@ -136,107 +254,3 @@ spec:
           path: /var/lib/filebeat-data
           type: DirectoryOrCreate
 ---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: filebeat
-subjects:
-- kind: ServiceAccount
-  name: filebeat
-  namespace: kube-system
-roleRef:
-  kind: ClusterRole
-  name: filebeat
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: filebeat
-  namespace: kube-system
-subjects:
-  - kind: ServiceAccount
-    name: filebeat
-    namespace: kube-system
-roleRef:
-  kind: Role
-  name: filebeat
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: filebeat-kubeadm-config
-  namespace: kube-system
-subjects:
-  - kind: ServiceAccount
-    name: filebeat
-    namespace: kube-system
-roleRef:
-  kind: Role
-  name: filebeat-kubeadm-config
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: filebeat
-  labels:
-    k8s-app: filebeat
-rules:
-- apiGroups: [""] # "" indicates the core API group
-  resources:
-  - namespaces
-  - pods
-  - nodes
-  verbs:
-  - get
-  - watch
-  - list
-- apiGroups: ["apps"]
-  resources:
-    - replicasets
-  verbs: ["get", "list", "watch"]
-- apiGroups: ["batch"]
-  resources:
-    - jobs
-  verbs: ["get", "list", "watch"]
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: filebeat
-  # should be the namespace where filebeat is running
-  namespace: kube-system
-  labels:
-    k8s-app: filebeat
-rules:
-  - apiGroups:
-      - coordination.k8s.io
-    resources:
-      - leases
-    verbs: ["get", "create", "update"]
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: filebeat-kubeadm-config
-  namespace: kube-system
-  labels:
-    k8s-app: filebeat
-rules:
-  - apiGroups: [""]
-    resources:
-      - configmaps
-    resourceNames:
-      - kubeadm-config
-    verbs: ["get"]
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: filebeat
-  namespace: kube-system
-  labels:
-    k8s-app: filebeat
----