From 7e4b93d3cbe819cf83c54cf3f7cc59e034880f1d Mon Sep 17 00:00:00 2001
From: Paolo Chila <paolo.chila@elastic.co>
Date: Wed, 29 Jan 2025 11:44:56 +0100
Subject: [PATCH] Fix k8s container logs input id using container names

---
 .../helm/elastic-agent/examples/eck/rendered/manifest.yaml  | 4 ++--
 .../examples/kubernetes-default/rendered/manifest.yaml      | 6 +++---
 .../kubernetes-hints-autodiscover/rendered/manifest.yaml    | 6 +++---
 .../examples/kubernetes-ksm-sharding/rendered/manifest.yaml | 6 +++---
 .../examples/kubernetes-only-logs/rendered/manifest.yaml    | 6 +++---
 .../examples/multiple-integrations/rendered/manifest.yaml   | 6 +++---
 .../examples/user-service-account/rendered/manifest.yaml    | 6 +++---
 .../base/elastic-agent-standalone-daemonset-configmap.yaml  | 4 ++--
 .../elastic-agent-standalone-ksm-daemonset-configmap.yaml   | 4 ++--
 .../elastic-agent-standalone-ksm-daemonset-configmap.yaml   | 4 ++--
 deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml  | 4 ++--
 .../elastic-agent-standalone-daemonset-configmap.yaml       | 6 +++---
 .../elastic-agent-standalone-ksm-daemonset-configmap.yaml   | 4 ++--
 .../base/elastic-agent-standalone/config-map.yaml           | 4 ++--
 .../pkg/agent/application/coordinator/testdata/config.yaml  | 6 +++---
 15 files changed, 38 insertions(+), 38 deletions(-)

diff --git a/deploy/helm/elastic-agent/examples/eck/rendered/manifest.yaml b/deploy/helm/elastic-agent/examples/eck/rendered/manifest.yaml
index 0a94afa4690..855c209b4dc 100644
--- a/deploy/helm/elastic-agent/examples/eck/rendered/manifest.yaml
+++ b/deploy/helm/elastic-agent/examples/eck/rendered/manifest.yaml
@@ -321,13 +321,13 @@ stringData:
         - data_stream:
             dataset: kubernetes.container_logs
             type: logs
-          id: kubernetes-container-logs-${kubernetes.pod.name}-${kubernetes.container.id}
+          id: kubernetes-container-logs-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
           parsers:
           - container:
               format: auto
               stream: all
           paths:
-          - /var/log/containers/*${kubernetes.container.id}.log
+          - /var/log/pods/${kubernetes.namespace}_${kubernetes.pod.name}_${kubernetes.pod.uid}/${kubernetes.container.name}/*.log
           processors:
           - add_fields:
               fields:
diff --git a/deploy/helm/elastic-agent/examples/kubernetes-default/rendered/manifest.yaml b/deploy/helm/elastic-agent/examples/kubernetes-default/rendered/manifest.yaml
index d5c609a4c2b..9b303c9d292 100644
--- a/deploy/helm/elastic-agent/examples/kubernetes-default/rendered/manifest.yaml
+++ b/deploy/helm/elastic-agent/examples/kubernetes-default/rendered/manifest.yaml
@@ -320,13 +320,13 @@ stringData:
         - data_stream:
             dataset: kubernetes.container_logs
             type: logs
-          id: kubernetes-container-logs-${kubernetes.pod.name}-${kubernetes.container.id}
+          id: kubernetes-container-logs-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
           parsers:
           - container:
               format: auto
               stream: all
           paths:
-          - /var/log/containers/*${kubernetes.container.id}.log
+          - /var/log/pods/${kubernetes.namespace}_${kubernetes.pod.name}_${kubernetes.pod.uid}/${kubernetes.container.name}/*.log
           processors:
           - add_fields:
               fields:
@@ -888,7 +888,7 @@ spec:
       labels:
         name: agent-pernode-example
       annotations:
-        checksum/config: 233affcd72143e637a130b5f099c30e194d90042eb00a26512f51c844c65a821
+        checksum/config: 0a02c8b3783f5e072f6eb62d87c3fefb2df9cdaf8efb7dbda9bcc11050246c3d
     spec:
       automountServiceAccountToken: true
       containers:
diff --git a/deploy/helm/elastic-agent/examples/kubernetes-hints-autodiscover/rendered/manifest.yaml b/deploy/helm/elastic-agent/examples/kubernetes-hints-autodiscover/rendered/manifest.yaml
index 87beb58f128..e85b265f58f 100644
--- a/deploy/helm/elastic-agent/examples/kubernetes-hints-autodiscover/rendered/manifest.yaml
+++ b/deploy/helm/elastic-agent/examples/kubernetes-hints-autodiscover/rendered/manifest.yaml
@@ -320,13 +320,13 @@ stringData:
         - data_stream:
             dataset: kubernetes.container_logs
             type: logs
-          id: kubernetes-container-logs-${kubernetes.pod.name}-${kubernetes.container.id}
+          id: kubernetes-container-logs-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
           parsers:
           - container:
               format: auto
               stream: all
           paths:
-          - /var/log/containers/*${kubernetes.container.id}.log
+          - /var/log/pods/${kubernetes.namespace}_${kubernetes.pod.name}_${kubernetes.pod.uid}/${kubernetes.container.name}/*.log
           processors:
           - add_fields:
               fields:
@@ -891,7 +891,7 @@ spec:
       labels:
         name: agent-pernode-example
       annotations:
-        checksum/config: daca0d998edb3afa587d96e69b0833f6919ca6ba72f58f3a1f83b22d7e5ffaf6
+        checksum/config: 440922798e9423e250298ce729018751f6a9d3343a03f3e54f93380e932acdfe
     spec:
       automountServiceAccountToken: true
       containers:
diff --git a/deploy/helm/elastic-agent/examples/kubernetes-ksm-sharding/rendered/manifest.yaml b/deploy/helm/elastic-agent/examples/kubernetes-ksm-sharding/rendered/manifest.yaml
index 1993d07a8ef..fb3bd78e505 100644
--- a/deploy/helm/elastic-agent/examples/kubernetes-ksm-sharding/rendered/manifest.yaml
+++ b/deploy/helm/elastic-agent/examples/kubernetes-ksm-sharding/rendered/manifest.yaml
@@ -132,13 +132,13 @@ stringData:
         - data_stream:
             dataset: kubernetes.container_logs
             type: logs
-          id: kubernetes-container-logs-${kubernetes.pod.name}-${kubernetes.container.id}
+          id: kubernetes-container-logs-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
           parsers:
           - container:
               format: auto
               stream: all
           paths:
-          - /var/log/containers/*${kubernetes.container.id}.log
+          - /var/log/pods/${kubernetes.namespace}_${kubernetes.pod.name}_${kubernetes.pod.uid}/${kubernetes.container.name}/*.log
           processors:
           - add_fields:
               fields:
@@ -949,7 +949,7 @@ spec:
       labels:
         name: agent-pernode-example
       annotations:
-        checksum/config: 05797fdfdd3cdeefb99e39e0f4756a6b812465509b31195ff57ae3925aa5e087
+        checksum/config: 916336da56b9ea407c8b613a721335c42af4fb6df724f54d0f301e6c193b8e4e
     spec:
       automountServiceAccountToken: true
       containers:
diff --git a/deploy/helm/elastic-agent/examples/kubernetes-only-logs/rendered/manifest.yaml b/deploy/helm/elastic-agent/examples/kubernetes-only-logs/rendered/manifest.yaml
index 599b020ad40..548fa32afcc 100644
--- a/deploy/helm/elastic-agent/examples/kubernetes-only-logs/rendered/manifest.yaml
+++ b/deploy/helm/elastic-agent/examples/kubernetes-only-logs/rendered/manifest.yaml
@@ -49,13 +49,13 @@ stringData:
         - data_stream:
             dataset: kubernetes.container_logs
             type: logs
-          id: kubernetes-container-logs-${kubernetes.pod.name}-${kubernetes.container.id}
+          id: kubernetes-container-logs-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
           parsers:
           - container:
               format: auto
               stream: all
           paths:
-          - /var/log/containers/*${kubernetes.container.id}.log
+          - /var/log/pods/${kubernetes.namespace}_${kubernetes.pod.name}_${kubernetes.pod.uid}/${kubernetes.container.name}/*.log
           processors:
           - add_fields:
               fields:
@@ -223,7 +223,7 @@ spec:
       labels:
         name: agent-pernode-example
       annotations:
-        checksum/config: 0840dcdf026f64cefb7aa69f420bc923d7e2d7d6e9a239e107fd2684e309d8ae
+        checksum/config: 7c35f814b93d366f98c472a618bd968083df7c07072b6e510cbb71ad26b0b6ab
     spec:
       automountServiceAccountToken: true
       containers:
diff --git a/deploy/helm/elastic-agent/examples/multiple-integrations/rendered/manifest.yaml b/deploy/helm/elastic-agent/examples/multiple-integrations/rendered/manifest.yaml
index 1acea05f377..31de9a04ce2 100644
--- a/deploy/helm/elastic-agent/examples/multiple-integrations/rendered/manifest.yaml
+++ b/deploy/helm/elastic-agent/examples/multiple-integrations/rendered/manifest.yaml
@@ -346,13 +346,13 @@ stringData:
         - data_stream:
             dataset: kubernetes.container_logs
             type: logs
-          id: kubernetes-container-logs-${kubernetes.pod.name}-${kubernetes.container.id}
+          id: kubernetes-container-logs-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
           parsers:
           - container:
               format: auto
               stream: all
           paths:
-          - /var/log/containers/*${kubernetes.container.id}.log
+          - /var/log/pods/${kubernetes.namespace}_${kubernetes.pod.name}_${kubernetes.pod.uid}/${kubernetes.container.name}/*.log
           processors:
           - add_fields:
               fields:
@@ -917,7 +917,7 @@ spec:
       labels:
         name: agent-pernode-example
       annotations:
-        checksum/config: daca0d998edb3afa587d96e69b0833f6919ca6ba72f58f3a1f83b22d7e5ffaf6
+        checksum/config: 440922798e9423e250298ce729018751f6a9d3343a03f3e54f93380e932acdfe
     spec:
       automountServiceAccountToken: true
       containers:
diff --git a/deploy/helm/elastic-agent/examples/user-service-account/rendered/manifest.yaml b/deploy/helm/elastic-agent/examples/user-service-account/rendered/manifest.yaml
index 46f2be3a360..7e792044e67 100644
--- a/deploy/helm/elastic-agent/examples/user-service-account/rendered/manifest.yaml
+++ b/deploy/helm/elastic-agent/examples/user-service-account/rendered/manifest.yaml
@@ -296,13 +296,13 @@ stringData:
         - data_stream:
             dataset: kubernetes.container_logs
             type: logs
-          id: kubernetes-container-logs-${kubernetes.pod.name}-${kubernetes.container.id}
+          id: kubernetes-container-logs-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
           parsers:
           - container:
               format: auto
               stream: all
           paths:
-          - /var/log/containers/*${kubernetes.container.id}.log
+          - /var/log/pods/${kubernetes.namespace}_${kubernetes.pod.name}_${kubernetes.pod.uid}/${kubernetes.container.name}/*.log
           processors:
           - add_fields:
               fields:
@@ -868,7 +868,7 @@ spec:
       labels:
         name: agent-pernode-example
       annotations:
-        checksum/config: 233affcd72143e637a130b5f099c30e194d90042eb00a26512f51c844c65a821
+        checksum/config: 0a02c8b3783f5e072f6eb62d87c3fefb2df9cdaf8efb7dbda9bcc11050246c3d
     spec:
       automountServiceAccountToken: true
       containers:
diff --git a/deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-standalone/base/elastic-agent-standalone-daemonset-configmap.yaml b/deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-standalone/base/elastic-agent-standalone-daemonset-configmap.yaml
index 8e0f0aef01f..08b19853c03 100644
--- a/deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-standalone/base/elastic-agent-standalone-daemonset-configmap.yaml
+++ b/deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-standalone/base/elastic-agent-standalone-daemonset-configmap.yaml
@@ -354,7 +354,7 @@ data:
             condition: '${host.platform} == ''windows'''
             ignore_older: 72h
       # Input ID allowing Elastic Agent to track the state of this input. Must be unique.
-      - id: container-log-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.id}
+      - id: container-log-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
         type: filestream
         use_output: default
         meta:
@@ -366,7 +366,7 @@ data:
         streams:
           # Stream ID for this data stream allowing Filebeat to track the state of the ingested files. Must be unique.
           # Each filestream data stream creates a separate instance of the Filebeat filestream input.
-          - id: container-log-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.id}
+          - id: container-log-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
             data_stream:
               dataset: kubernetes.container_logs
               type: logs
diff --git a/deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-standalone/base/elastic-agent-standalone-ksm-daemonset-configmap.yaml b/deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-standalone/base/elastic-agent-standalone-ksm-daemonset-configmap.yaml
index 9bfae8731c4..0ce3f7c5bdc 100644
--- a/deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-standalone/base/elastic-agent-standalone-ksm-daemonset-configmap.yaml
+++ b/deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-standalone/base/elastic-agent-standalone-ksm-daemonset-configmap.yaml
@@ -122,7 +122,7 @@ data:
               dataset: system.system
             condition: '${host.platform} == ''windows'''
             ignore_older: 72h
-      - id: container-log-${kubernetes.pod.name}-${kubernetes.container.id}
+      - id: container-log-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
         type: filestream
         use_output: default
         meta:
@@ -146,7 +146,7 @@ data:
               #     negate: true
               #     match: after
             paths:
-              - /var/log/containers/*${kubernetes.container.id}.log
+              - /var/log/pods/${kubernetes.namespace}_${kubernetes.pod.name}_${kubernetes.pod.uid}/${kubernetes.container.name}/*.log
       - id: audit-log
         type: filestream
         use_output: default
diff --git a/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-standalone/base/elastic-agent-standalone-ksm-daemonset-configmap.yaml b/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-standalone/base/elastic-agent-standalone-ksm-daemonset-configmap.yaml
index 9bfae8731c4..0ce3f7c5bdc 100644
--- a/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-standalone/base/elastic-agent-standalone-ksm-daemonset-configmap.yaml
+++ b/deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-standalone/base/elastic-agent-standalone-ksm-daemonset-configmap.yaml
@@ -122,7 +122,7 @@ data:
               dataset: system.system
             condition: '${host.platform} == ''windows'''
             ignore_older: 72h
-      - id: container-log-${kubernetes.pod.name}-${kubernetes.container.id}
+      - id: container-log-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
         type: filestream
         use_output: default
         meta:
@@ -146,7 +146,7 @@ data:
               #     negate: true
               #     match: after
             paths:
-              - /var/log/containers/*${kubernetes.container.id}.log
+              - /var/log/pods/${kubernetes.namespace}_${kubernetes.pod.name}_${kubernetes.pod.uid}/${kubernetes.container.name}/*.log
       - id: audit-log
         type: filestream
         use_output: default
diff --git a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml
index 9b486b7ab6f..bedd721bd79 100644
--- a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml
+++ b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml
@@ -351,7 +351,7 @@ data:
             condition: '${host.platform} == ''windows'''
             ignore_older: 72h
       # Input ID allowing Elastic Agent to track the state of this input. Must be unique.
-      - id: container-log-${kubernetes.pod.name}-${kubernetes.container.id}
+      - id: container-log-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
         type: filestream
         use_output: default
         meta:
@@ -363,7 +363,7 @@ data:
         streams:
           # Stream ID for this data stream allowing Filebeat to track the state of the ingested files. Must be unique.
           # Each filestream data stream creates a separate instance of the Filebeat filestream input.
-          - id: container-log-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.id}
+          - id: container-log-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
             data_stream:
               dataset: kubernetes.container_logs
               type: logs
diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml
index 4d837d4775b..9ddd132176c 100644
--- a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml
+++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml
@@ -351,7 +351,7 @@ data:
             condition: '${host.platform} == ''windows'''
             ignore_older: 72h
       # Input ID allowing Elastic Agent to track the state of this input. Must be unique.
-      - id: container-log-${kubernetes.pod.name}-${kubernetes.container.id}
+      - id: container-log-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
         type: filestream
         use_output: default
         meta:
@@ -363,7 +363,7 @@ data:
         streams:
           # Stream ID for this data stream allowing Filebeat to track the state of the ingested files. Must be unique.
           # Each filestream data stream creates a separate instance of the Filebeat filestream input.
-          - id: container-log-${kubernetes.pod.name}-${kubernetes.container.id}
+          - id: container-log-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
             data_stream:
               dataset: kubernetes.container_logs
               type: logs
@@ -378,7 +378,7 @@ data:
               #     negate: true
               #     match: after
             paths:
-              - /var/log/containers/*${kubernetes.container.id}.log
+              - /var/log/pods/${kubernetes.namespace}_${kubernetes.pod.name}_${kubernetes.pod.uid}/${kubernetes.container.name}/*.log
       - id: audit-log
         type: filestream
         use_output: default
diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-ksm-daemonset-configmap.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-ksm-daemonset-configmap.yaml
index 9bfae8731c4..0ce3f7c5bdc 100644
--- a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-ksm-daemonset-configmap.yaml
+++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-ksm-daemonset-configmap.yaml
@@ -122,7 +122,7 @@ data:
               dataset: system.system
             condition: '${host.platform} == ''windows'''
             ignore_older: 72h
-      - id: container-log-${kubernetes.pod.name}-${kubernetes.container.id}
+      - id: container-log-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
         type: filestream
         use_output: default
         meta:
@@ -146,7 +146,7 @@ data:
               #     negate: true
               #     match: after
             paths:
-              - /var/log/containers/*${kubernetes.container.id}.log
+              - /var/log/pods/${kubernetes.namespace}_${kubernetes.pod.name}_${kubernetes.pod.uid}/${kubernetes.container.name}/*.log
       - id: audit-log
         type: filestream
         use_output: default
diff --git a/dev-tools/kubernetes/base/elastic-agent-standalone/config-map.yaml b/dev-tools/kubernetes/base/elastic-agent-standalone/config-map.yaml
index 7920f339a8a..593bc9be8ab 100644
--- a/dev-tools/kubernetes/base/elastic-agent-standalone/config-map.yaml
+++ b/dev-tools/kubernetes/base/elastic-agent-standalone/config-map.yaml
@@ -330,7 +330,7 @@ data:
               dataset: system.system
             condition: '${host.platform} == ''windows'''
             ignore_older: 72h
-      - id: container-log-${kubernetes.pod.name}-${kubernetes.container.id}
+      - id: container-log-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
         type: filestream
         use_output: default
         meta:
@@ -354,7 +354,7 @@ data:
               #     negate: true
               #     match: after
             paths:
-              - /var/log/containers/*${kubernetes.container.id}.log
+              - /var/log/pods/${kubernetes.namespace}_${kubernetes.pod.name}_${kubernetes.pod.uid}/${kubernetes.container.name}/*.log
       - id: audit-log
         type: filestream
         use_output: default
diff --git a/internal/pkg/agent/application/coordinator/testdata/config.yaml b/internal/pkg/agent/application/coordinator/testdata/config.yaml
index bbae47276ae..73204a45a2d 100644
--- a/internal/pkg/agent/application/coordinator/testdata/config.yaml
+++ b/internal/pkg/agent/application/coordinator/testdata/config.yaml
@@ -243,7 +243,7 @@ inputs:
           dataset: system.system
         condition: '${host.platform} == ''windows'''
         ignore_older: 72h
-  - id: container-log-${kubernetes.pod.name}-${kubernetes.container.id}
+  - id: container-log-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
     type: filestream
     use_output: default
     meta:
@@ -253,7 +253,7 @@ inputs:
     data_stream:
       namespace: default
     streams:
-      - id: container-log-${kubernetes.pod.name}-${kubernetes.container.id}
+      - id: container-log-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
         data_stream:
           dataset: kubernetes.container_logs
           type: logs
@@ -261,7 +261,7 @@ inputs:
         parsers:
           - container: ~
         paths:
-          - /var/log/containers/*${kubernetes.container.id}.log
+          - /var/log/pods/${kubernetes.namespace}_${kubernetes.pod.name}_${kubernetes.pod.uid}/${kubernetes.container.name}/*.log
   - id: audit-log
     type: filestream
     use_output: default