[O11y][Hadoop] Migrate Hadoop package to adopt ecs@mappings (elastic#…

…10407) * [hadoop] - change to ECS version [email protected] * update pr link in changelog.yml
taylor-swanson · Jul 10, 2024 · 52fb472 · 52fb472
1 parent 033f5be
commit 52fb472
Show file tree

Hide file tree

Showing 25 changed files with 81 additions and 169 deletions.
diff --git a/packages/hadoop/_dev/build/build.yml b/packages/hadoop/_dev/build/build.yml
@@ -1,3 +1,3 @@
 dependencies:
   ecs:
-    reference: git@v8.5.1
+    reference: "git@v8.11.0"
diff --git a/packages/hadoop/_dev/build/docs/README.md b/packages/hadoop/_dev/build/docs/README.md
@@ -25,6 +25,10 @@ This data stream collects Application metrics.
 
 {{event "application"}}
 
+**ECS Field Reference**
+
+Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields.
+
 {{fields "application"}}
 
 ## cluster
@@ -33,6 +37,10 @@ This data stream collects Cluster metrics.
 
 {{event "cluster"}}
 
+**ECS Field Reference**
+
+Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields.
+
 {{fields "cluster"}}
 
 ## datanode
@@ -41,6 +49,10 @@ This data stream collects Datanode metrics.
 
 {{event "datanode"}}
 
+**ECS Field Reference**
+
+Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields.
+
 {{fields "datanode"}}
 
 ## namenode
@@ -49,11 +61,19 @@ This data stream collects Namenode metrics.
 
 {{event "namenode"}}
 
+**ECS Field Reference**
+
+Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields.
+
 {{fields "namenode"}}
 ## node_manager
 
 This data stream collects Node Manager metrics.
 
 {{event "node_manager"}}
 
+**ECS Field Reference**
+
+Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields.
+
 {{fields "node_manager"}}
diff --git a/packages/hadoop/changelog.yml b/packages/hadoop/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.7.0"
+  changes:
+    - description: ECS version updated to 8.11.0. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10407
 - version: "1.6.0"
   changes:
     - description: Add global filter on data_stream.dataset to improve performance.

diff --git a/...oop/data_stream/application/_dev/test/pipeline/test-application-metrics.log-expected.json b/...oop/data_stream/application/_dev/test/pipeline/test-application-metrics.log-expected.json
@@ -2,7 +2,7 @@
     "expected": [
         {
             "ecs": {
-                "version": "8.5.1"
+                "version": "8.11.0"
             },
             "event": {
                 "category": [
@@ -39,7 +39,7 @@
         },
         {
             "ecs": {
-                "version": "8.5.1"
+                "version": "8.11.0"
             },
             "event": {
                 "category": [

diff --git a/packages/hadoop/data_stream/application/elasticsearch/ingest_pipeline/default.yml b/packages/hadoop/data_stream/application/elasticsearch/ingest_pipeline/default.yml
@@ -3,7 +3,7 @@ description: Pipeline for parsing Hadoop Application metrics
 processors:
   - set:
       field: ecs.version
-      value: '8.5.1'
+      value: '8.11.0'
   - set:
       field: event.type
       value: [info]

diff --git a/packages/hadoop/data_stream/application/fields/ecs.yml b/packages/hadoop/data_stream/application/fields/ecs.yml
diff --git a/packages/hadoop/data_stream/application/sample_event.json b/packages/hadoop/data_stream/application/sample_event.json
@@ -13,7 +13,7 @@
         "type": "logs"
     },
     "ecs": {
-        "version": "8.5.1"
+        "version": "8.11.0"
     },
     "elastic_agent": {
         "id": "2d054344-10a6-40d9-90c1-ea017fecfda3",

diff --git a/...ges/hadoop/data_stream/cluster/_dev/test/pipeline/test-cluster-metrics.json-expected.json b/...ges/hadoop/data_stream/cluster/_dev/test/pipeline/test-cluster-metrics.json-expected.json
@@ -3,7 +3,7 @@
         {
             "@timestamp": "2016-10-25T12:49:34.000Z",
             "ecs": {
-                "version": "8.5.1"
+                "version": "8.11.0"
             },
             "event": {
                 "category": [
@@ -35,7 +35,7 @@
         {
             "@timestamp": "2016-10-25T12:49:34.000Z",
             "ecs": {
-                "version": "8.5.1"
+                "version": "8.11.0"
             },
             "event": {
                 "category": [

diff --git a/packages/hadoop/data_stream/cluster/elasticsearch/ingest_pipeline/default.yml b/packages/hadoop/data_stream/cluster/elasticsearch/ingest_pipeline/default.yml
@@ -3,7 +3,7 @@ description: Pipeline for parsing Hadoop Cluster metrics
 processors:
   - set:
       field: ecs.version
-      value: '8.5.1'
+      value: '8.11.0'
   - set:
       field: event.type
       value: [info]

diff --git a/packages/hadoop/data_stream/cluster/fields/ecs.yml b/packages/hadoop/data_stream/cluster/fields/ecs.yml
@@ -1,24 +1,6 @@
-- external: ecs
-  name: ecs.version
-- external: ecs
-  name: event.category
-- external: ecs
-  name: event.dataset
-- external: ecs
-  name: event.kind
-- external: ecs
-  name: event.module
-- external: ecs
-  name: event.type
-- external: ecs
-  name: host.ip
 - external: ecs
   name: service.address
   dimension: true
-- external: ecs
-  name: service.type
-- external: ecs
-  name: tags
 - external: ecs
   name: cloud.instance.id
   dimension: true

diff --git a/packages/hadoop/data_stream/cluster/sample_event.json b/packages/hadoop/data_stream/cluster/sample_event.json
@@ -13,7 +13,7 @@
         "type": "metrics"
     },
     "ecs": {
-        "version": "8.5.1"
+        "version": "8.11.0"
     },
     "elastic_agent": {
         "id": "abf8f8c1-f293-4e16-a8f8-8cf48014d040",
@@ -59,7 +59,7 @@
             "172.27.0.7"
         ],
         "mac": [
-            "02:42:ac:1b:00:07"
+            "02-42-AC-1F-00-07"
         ],
         "name": "docker-fleet-agent",
         "os": {

diff --git a/...s/hadoop/data_stream/datanode/_dev/test/pipeline/test-datanode-metrics.json-expected.json b/...s/hadoop/data_stream/datanode/_dev/test/pipeline/test-datanode-metrics.json-expected.json
@@ -3,7 +3,7 @@
         {
             "@timestamp": "2016-10-25T12:49:34.000Z",
             "ecs": {
-                "version": "8.5.1"
+                "version": "8.11.0"
             },
             "event": {
                 "category": [
@@ -43,7 +43,7 @@
         {
             "@timestamp": "2016-10-25T12:50:34.000Z",
             "ecs": {
-                "version": "8.5.1"
+                "version": "8.11.0"
             },
             "event": {
                 "category": [

diff --git a/packages/hadoop/data_stream/datanode/elasticsearch/ingest_pipeline/default.yml b/packages/hadoop/data_stream/datanode/elasticsearch/ingest_pipeline/default.yml
@@ -3,7 +3,7 @@ description: Pipeline for parsing Hadoop Datanode metrics
 processors:
   - set:
       field: ecs.version
-      value: '8.5.1'
+      value: '8.11.0'
   - set:
       field: event.type
       value: [info]

diff --git a/packages/hadoop/data_stream/datanode/fields/ecs.yml b/packages/hadoop/data_stream/datanode/fields/ecs.yml
@@ -1,24 +1,6 @@
-- external: ecs
-  name: ecs.version
-- external: ecs
-  name: event.category
-- external: ecs
-  name: event.dataset
-- external: ecs
-  name: event.kind
-- external: ecs
-  name: event.module
-- external: ecs
-  name: event.type
-- external: ecs
-  name: host.ip
 - external: ecs
   name: service.address
   dimension: true
-- external: ecs
-  name: service.type
-- external: ecs
-  name: tags
 - external: ecs
   name: cloud.instance.id
   dimension: true

diff --git a/packages/hadoop/data_stream/datanode/sample_event.json b/packages/hadoop/data_stream/datanode/sample_event.json
@@ -13,7 +13,7 @@
         "type": "metrics"
     },
     "ecs": {
-        "version": "8.5.1"
+        "version": "8.11.0"
     },
     "elastic_agent": {
         "id": "2d054344-10a6-40d9-90c1-ea017fecfda3",

diff --git a/...s/hadoop/data_stream/namenode/_dev/test/pipeline/test-namenode-metrics.json-expected.json b/...s/hadoop/data_stream/namenode/_dev/test/pipeline/test-namenode-metrics.json-expected.json
@@ -3,7 +3,7 @@
         {
             "@timestamp": "2016-10-25T12:49:34.000Z",
             "ecs": {
-                "version": "8.5.1"
+                "version": "8.11.0"
             },
             "event": {
                 "category": [

diff --git a/packages/hadoop/data_stream/namenode/elasticsearch/ingest_pipeline/default.yml b/packages/hadoop/data_stream/namenode/elasticsearch/ingest_pipeline/default.yml
@@ -3,7 +3,7 @@ description: Pipeline for parsing Hadoop Namenode metrics
 processors:
   - set:
       field: ecs.version
-      value: '8.5.1'
+      value: '8.11.0'
   - set:
       field: event.type
       value: [info]

diff --git a/packages/hadoop/data_stream/namenode/fields/ecs.yml b/packages/hadoop/data_stream/namenode/fields/ecs.yml
@@ -1,24 +1,6 @@
-- external: ecs
-  name: ecs.version
-- external: ecs
-  name: event.category
-- external: ecs
-  name: event.dataset
-- external: ecs
-  name: event.kind
-- external: ecs
-  name: event.module
-- external: ecs
-  name: event.type
-- external: ecs
-  name: host.ip
 - external: ecs
   name: service.address
   dimension: true
-- external: ecs
-  name: service.type
-- external: ecs
-  name: tags
 - external: ecs
   name: cloud.instance.id
   dimension: true

diff --git a/packages/hadoop/data_stream/namenode/sample_event.json b/packages/hadoop/data_stream/namenode/sample_event.json
@@ -13,7 +13,7 @@
         "type": "metrics"
     },
     "ecs": {
-        "version": "8.5.1"
+        "version": "8.11.0"
     },
     "elastic_agent": {
         "id": "adf6847a-3726-4fe6-a202-147021ff3cbc",
@@ -74,7 +74,7 @@
             "192.168.160.7"
         ],
         "mac": [
-            "02:42:c0:a8:a0:07"
+            "02-42-AC-1F-00-07"
         ],
         "name": "docker-fleet-agent",
         "os": {

diff --git a/.../data_stream/node_manager/_dev/test/pipeline/test-node-manager-metrics.json-expected.json b/.../data_stream/node_manager/_dev/test/pipeline/test-node-manager-metrics.json-expected.json
@@ -3,7 +3,7 @@
         {
             "@timestamp": "2016-10-25T12:49:34.000Z",
             "ecs": {
-                "version": "8.5.1"
+                "version": "8.11.0"
             },
             "event": {
                 "category": [

diff --git a/packages/hadoop/data_stream/node_manager/elasticsearch/ingest_pipeline/default.yml b/packages/hadoop/data_stream/node_manager/elasticsearch/ingest_pipeline/default.yml
@@ -3,7 +3,7 @@ description: Pipeline for parsing Hadoop Datanode metrics
 processors:
   - set:
       field: ecs.version
-      value: '8.5.1'
+      value: '8.11.0'
   - set:
       field: event.type
       value: [info]

diff --git a/packages/hadoop/data_stream/node_manager/fields/ecs.yml b/packages/hadoop/data_stream/node_manager/fields/ecs.yml
@@ -1,24 +1,6 @@
-- external: ecs
-  name: ecs.version
-- external: ecs
-  name: event.category
-- external: ecs
-  name: event.dataset
-- external: ecs
-  name: event.kind
-- external: ecs
-  name: event.module
-- external: ecs
-  name: event.type
-- external: ecs
-  name: host.ip
 - external: ecs
   name: service.address
   dimension: true
-- external: ecs
-  name: service.type
-- external: ecs
-  name: tags
 - external: ecs
   name: cloud.instance.id
   dimension: true

diff --git a/packages/hadoop/data_stream/node_manager/sample_event.json b/packages/hadoop/data_stream/node_manager/sample_event.json
@@ -13,7 +13,7 @@
         "type": "metrics"
     },
     "ecs": {
-        "version": "8.5.1"
+        "version": "8.11.0"
     },
     "elastic_agent": {
         "id": "adf6847a-3726-4fe6-a202-147021ff3cbc",
@@ -57,7 +57,7 @@
             "192.168.160.7"
         ],
         "mac": [
-            "02:42:c0:a8:a0:07"
+            "02-42-AC-1F-00-07"
         ],
         "name": "docker-fleet-agent",
         "os": {