diff --git a/packages/hadoop/_dev/build/build.yml b/packages/hadoop/_dev/build/build.yml index aaafc5d833b..2bfcfc223b0 100644 --- a/packages/hadoop/_dev/build/build.yml +++ b/packages/hadoop/_dev/build/build.yml @@ -1,3 +1,3 @@ dependencies: ecs: - reference: git@v8.5.1 + reference: "git@v8.11.0" diff --git a/packages/hadoop/_dev/build/docs/README.md b/packages/hadoop/_dev/build/docs/README.md index 0b0cafc92aa..9c44fa2ab63 100644 --- a/packages/hadoop/_dev/build/docs/README.md +++ b/packages/hadoop/_dev/build/docs/README.md @@ -25,6 +25,10 @@ This data stream collects Application metrics. {{event "application"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "application"}} ## cluster @@ -33,6 +37,10 @@ This data stream collects Cluster metrics. {{event "cluster"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "cluster"}} ## datanode @@ -41,6 +49,10 @@ This data stream collects Datanode metrics. {{event "datanode"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "datanode"}} ## namenode @@ -49,6 +61,10 @@ This data stream collects Namenode metrics. {{event "namenode"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "namenode"}} ## node_manager @@ -56,4 +72,8 @@ This data stream collects Node Manager metrics. {{event "node_manager"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "node_manager"}} diff --git a/packages/hadoop/changelog.yml b/packages/hadoop/changelog.yml index 5572e2d11d7..d5386d64dde 100644 --- a/packages/hadoop/changelog.yml +++ b/packages/hadoop/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.7.0" + changes: + - description: ECS version updated to 8.11.0. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10407 - version: "1.6.0" changes: - description: Add global filter on data_stream.dataset to improve performance. diff --git a/packages/hadoop/data_stream/application/_dev/test/pipeline/test-application-metrics.log-expected.json b/packages/hadoop/data_stream/application/_dev/test/pipeline/test-application-metrics.log-expected.json index cbc4d609698..1b997e3ea5d 100644 --- a/packages/hadoop/data_stream/application/_dev/test/pipeline/test-application-metrics.log-expected.json +++ b/packages/hadoop/data_stream/application/_dev/test/pipeline/test-application-metrics.log-expected.json @@ -2,7 +2,7 @@ "expected": [ { "ecs": { - "version": "8.5.1" + "version": "8.11.0" }, "event": { "category": [ @@ -39,7 +39,7 @@ }, { "ecs": { - "version": "8.5.1" + "version": "8.11.0" }, "event": { "category": [ diff --git a/packages/hadoop/data_stream/application/elasticsearch/ingest_pipeline/default.yml b/packages/hadoop/data_stream/application/elasticsearch/ingest_pipeline/default.yml index 8bed3ac8ccf..ade5b02cb6c 100644 --- a/packages/hadoop/data_stream/application/elasticsearch/ingest_pipeline/default.yml +++ b/packages/hadoop/data_stream/application/elasticsearch/ingest_pipeline/default.yml @@ -3,7 +3,7 @@ description: Pipeline for parsing Hadoop Application metrics processors: - set: field: ecs.version - value: '8.5.1' + value: '8.11.0' - set: field: event.type value: [info] diff --git a/packages/hadoop/data_stream/application/fields/ecs.yml b/packages/hadoop/data_stream/application/fields/ecs.yml deleted file mode 100644 index e07979260d0..00000000000 --- a/packages/hadoop/data_stream/application/fields/ecs.yml +++ /dev/null @@ -1,17 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.dataset -- external: ecs - name: event.kind -- external: ecs - name: event.module -- external: ecs - name: event.type -- external: ecs - name: host.ip -- external: ecs - name: tags - description: User defined tags. diff --git a/packages/hadoop/data_stream/application/sample_event.json b/packages/hadoop/data_stream/application/sample_event.json index d4d295a9824..53d675609a0 100644 --- a/packages/hadoop/data_stream/application/sample_event.json +++ b/packages/hadoop/data_stream/application/sample_event.json @@ -13,7 +13,7 @@ "type": "logs" }, "ecs": { - "version": "8.5.1" + "version": "8.11.0" }, "elastic_agent": { "id": "2d054344-10a6-40d9-90c1-ea017fecfda3", diff --git a/packages/hadoop/data_stream/cluster/_dev/test/pipeline/test-cluster-metrics.json-expected.json b/packages/hadoop/data_stream/cluster/_dev/test/pipeline/test-cluster-metrics.json-expected.json index 39401848952..298d9d21d68 100644 --- a/packages/hadoop/data_stream/cluster/_dev/test/pipeline/test-cluster-metrics.json-expected.json +++ b/packages/hadoop/data_stream/cluster/_dev/test/pipeline/test-cluster-metrics.json-expected.json @@ -3,7 +3,7 @@ { "@timestamp": "2016-10-25T12:49:34.000Z", "ecs": { - "version": "8.5.1" + "version": "8.11.0" }, "event": { "category": [ @@ -35,7 +35,7 @@ { "@timestamp": "2016-10-25T12:49:34.000Z", "ecs": { - "version": "8.5.1" + "version": "8.11.0" }, "event": { "category": [ diff --git a/packages/hadoop/data_stream/cluster/elasticsearch/ingest_pipeline/default.yml b/packages/hadoop/data_stream/cluster/elasticsearch/ingest_pipeline/default.yml index 1f8c98d5dfe..892d0aaa54a 100644 --- a/packages/hadoop/data_stream/cluster/elasticsearch/ingest_pipeline/default.yml +++ b/packages/hadoop/data_stream/cluster/elasticsearch/ingest_pipeline/default.yml @@ -3,7 +3,7 @@ description: Pipeline for parsing Hadoop Cluster metrics processors: - set: field: ecs.version - value: '8.5.1' + value: '8.11.0' - set: field: event.type value: [info] diff --git a/packages/hadoop/data_stream/cluster/fields/ecs.yml b/packages/hadoop/data_stream/cluster/fields/ecs.yml index e1068cd0714..2cfdfeb3103 100644 --- a/packages/hadoop/data_stream/cluster/fields/ecs.yml +++ b/packages/hadoop/data_stream/cluster/fields/ecs.yml @@ -1,24 +1,6 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.dataset -- external: ecs - name: event.kind -- external: ecs - name: event.module -- external: ecs - name: event.type -- external: ecs - name: host.ip - external: ecs name: service.address dimension: true -- external: ecs - name: service.type -- external: ecs - name: tags - external: ecs name: cloud.instance.id dimension: true diff --git a/packages/hadoop/data_stream/cluster/sample_event.json b/packages/hadoop/data_stream/cluster/sample_event.json index ba1a4f5f018..bee03c3b6eb 100644 --- a/packages/hadoop/data_stream/cluster/sample_event.json +++ b/packages/hadoop/data_stream/cluster/sample_event.json @@ -13,7 +13,7 @@ "type": "metrics" }, "ecs": { - "version": "8.5.1" + "version": "8.11.0" }, "elastic_agent": { "id": "abf8f8c1-f293-4e16-a8f8-8cf48014d040", @@ -59,7 +59,7 @@ "172.27.0.7" ], "mac": [ - "02:42:ac:1b:00:07" + "02-42-AC-1F-00-07" ], "name": "docker-fleet-agent", "os": { diff --git a/packages/hadoop/data_stream/datanode/_dev/test/pipeline/test-datanode-metrics.json-expected.json b/packages/hadoop/data_stream/datanode/_dev/test/pipeline/test-datanode-metrics.json-expected.json index 9ec829281bf..56be4515096 100644 --- a/packages/hadoop/data_stream/datanode/_dev/test/pipeline/test-datanode-metrics.json-expected.json +++ b/packages/hadoop/data_stream/datanode/_dev/test/pipeline/test-datanode-metrics.json-expected.json @@ -3,7 +3,7 @@ { "@timestamp": "2016-10-25T12:49:34.000Z", "ecs": { - "version": "8.5.1" + "version": "8.11.0" }, "event": { "category": [ @@ -43,7 +43,7 @@ { "@timestamp": "2016-10-25T12:50:34.000Z", "ecs": { - "version": "8.5.1" + "version": "8.11.0" }, "event": { "category": [ diff --git a/packages/hadoop/data_stream/datanode/elasticsearch/ingest_pipeline/default.yml b/packages/hadoop/data_stream/datanode/elasticsearch/ingest_pipeline/default.yml index 3aa7a215a4b..b0614032087 100644 --- a/packages/hadoop/data_stream/datanode/elasticsearch/ingest_pipeline/default.yml +++ b/packages/hadoop/data_stream/datanode/elasticsearch/ingest_pipeline/default.yml @@ -3,7 +3,7 @@ description: Pipeline for parsing Hadoop Datanode metrics processors: - set: field: ecs.version - value: '8.5.1' + value: '8.11.0' - set: field: event.type value: [info] diff --git a/packages/hadoop/data_stream/datanode/fields/ecs.yml b/packages/hadoop/data_stream/datanode/fields/ecs.yml index e1068cd0714..2cfdfeb3103 100644 --- a/packages/hadoop/data_stream/datanode/fields/ecs.yml +++ b/packages/hadoop/data_stream/datanode/fields/ecs.yml @@ -1,24 +1,6 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.dataset -- external: ecs - name: event.kind -- external: ecs - name: event.module -- external: ecs - name: event.type -- external: ecs - name: host.ip - external: ecs name: service.address dimension: true -- external: ecs - name: service.type -- external: ecs - name: tags - external: ecs name: cloud.instance.id dimension: true diff --git a/packages/hadoop/data_stream/datanode/sample_event.json b/packages/hadoop/data_stream/datanode/sample_event.json index 5b927836a14..bbdcb092f50 100644 --- a/packages/hadoop/data_stream/datanode/sample_event.json +++ b/packages/hadoop/data_stream/datanode/sample_event.json @@ -13,7 +13,7 @@ "type": "metrics" }, "ecs": { - "version": "8.5.1" + "version": "8.11.0" }, "elastic_agent": { "id": "2d054344-10a6-40d9-90c1-ea017fecfda3", diff --git a/packages/hadoop/data_stream/namenode/_dev/test/pipeline/test-namenode-metrics.json-expected.json b/packages/hadoop/data_stream/namenode/_dev/test/pipeline/test-namenode-metrics.json-expected.json index 836e4ec9d69..827693f3572 100644 --- a/packages/hadoop/data_stream/namenode/_dev/test/pipeline/test-namenode-metrics.json-expected.json +++ b/packages/hadoop/data_stream/namenode/_dev/test/pipeline/test-namenode-metrics.json-expected.json @@ -3,7 +3,7 @@ { "@timestamp": "2016-10-25T12:49:34.000Z", "ecs": { - "version": "8.5.1" + "version": "8.11.0" }, "event": { "category": [ diff --git a/packages/hadoop/data_stream/namenode/elasticsearch/ingest_pipeline/default.yml b/packages/hadoop/data_stream/namenode/elasticsearch/ingest_pipeline/default.yml index e043f474c3c..0adfabeb9a4 100644 --- a/packages/hadoop/data_stream/namenode/elasticsearch/ingest_pipeline/default.yml +++ b/packages/hadoop/data_stream/namenode/elasticsearch/ingest_pipeline/default.yml @@ -3,7 +3,7 @@ description: Pipeline for parsing Hadoop Namenode metrics processors: - set: field: ecs.version - value: '8.5.1' + value: '8.11.0' - set: field: event.type value: [info] diff --git a/packages/hadoop/data_stream/namenode/fields/ecs.yml b/packages/hadoop/data_stream/namenode/fields/ecs.yml index e1068cd0714..2cfdfeb3103 100644 --- a/packages/hadoop/data_stream/namenode/fields/ecs.yml +++ b/packages/hadoop/data_stream/namenode/fields/ecs.yml @@ -1,24 +1,6 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.dataset -- external: ecs - name: event.kind -- external: ecs - name: event.module -- external: ecs - name: event.type -- external: ecs - name: host.ip - external: ecs name: service.address dimension: true -- external: ecs - name: service.type -- external: ecs - name: tags - external: ecs name: cloud.instance.id dimension: true diff --git a/packages/hadoop/data_stream/namenode/sample_event.json b/packages/hadoop/data_stream/namenode/sample_event.json index b04e618a2ac..a1a7bbe72e9 100644 --- a/packages/hadoop/data_stream/namenode/sample_event.json +++ b/packages/hadoop/data_stream/namenode/sample_event.json @@ -13,7 +13,7 @@ "type": "metrics" }, "ecs": { - "version": "8.5.1" + "version": "8.11.0" }, "elastic_agent": { "id": "adf6847a-3726-4fe6-a202-147021ff3cbc", @@ -74,7 +74,7 @@ "192.168.160.7" ], "mac": [ - "02:42:c0:a8:a0:07" + "02-42-AC-1F-00-07" ], "name": "docker-fleet-agent", "os": { diff --git a/packages/hadoop/data_stream/node_manager/_dev/test/pipeline/test-node-manager-metrics.json-expected.json b/packages/hadoop/data_stream/node_manager/_dev/test/pipeline/test-node-manager-metrics.json-expected.json index ee7f7886040..76fdbff5744 100644 --- a/packages/hadoop/data_stream/node_manager/_dev/test/pipeline/test-node-manager-metrics.json-expected.json +++ b/packages/hadoop/data_stream/node_manager/_dev/test/pipeline/test-node-manager-metrics.json-expected.json @@ -3,7 +3,7 @@ { "@timestamp": "2016-10-25T12:49:34.000Z", "ecs": { - "version": "8.5.1" + "version": "8.11.0" }, "event": { "category": [ diff --git a/packages/hadoop/data_stream/node_manager/elasticsearch/ingest_pipeline/default.yml b/packages/hadoop/data_stream/node_manager/elasticsearch/ingest_pipeline/default.yml index e8b53e55bbb..4502a2ba803 100644 --- a/packages/hadoop/data_stream/node_manager/elasticsearch/ingest_pipeline/default.yml +++ b/packages/hadoop/data_stream/node_manager/elasticsearch/ingest_pipeline/default.yml @@ -3,7 +3,7 @@ description: Pipeline for parsing Hadoop Datanode metrics processors: - set: field: ecs.version - value: '8.5.1' + value: '8.11.0' - set: field: event.type value: [info] diff --git a/packages/hadoop/data_stream/node_manager/fields/ecs.yml b/packages/hadoop/data_stream/node_manager/fields/ecs.yml index e1068cd0714..2cfdfeb3103 100644 --- a/packages/hadoop/data_stream/node_manager/fields/ecs.yml +++ b/packages/hadoop/data_stream/node_manager/fields/ecs.yml @@ -1,24 +1,6 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.dataset -- external: ecs - name: event.kind -- external: ecs - name: event.module -- external: ecs - name: event.type -- external: ecs - name: host.ip - external: ecs name: service.address dimension: true -- external: ecs - name: service.type -- external: ecs - name: tags - external: ecs name: cloud.instance.id dimension: true diff --git a/packages/hadoop/data_stream/node_manager/sample_event.json b/packages/hadoop/data_stream/node_manager/sample_event.json index d10bbb010e0..50c9cfb8d8c 100644 --- a/packages/hadoop/data_stream/node_manager/sample_event.json +++ b/packages/hadoop/data_stream/node_manager/sample_event.json @@ -13,7 +13,7 @@ "type": "metrics" }, "ecs": { - "version": "8.5.1" + "version": "8.11.0" }, "elastic_agent": { "id": "adf6847a-3726-4fe6-a202-147021ff3cbc", @@ -57,7 +57,7 @@ "192.168.160.7" ], "mac": [ - "02:42:c0:a8:a0:07" + "02-42-AC-1F-00-07" ], "name": "docker-fleet-agent", "os": { diff --git a/packages/hadoop/docs/README.md b/packages/hadoop/docs/README.md index bce0513c7d9..70b381a3d3f 100644 --- a/packages/hadoop/docs/README.md +++ b/packages/hadoop/docs/README.md @@ -41,7 +41,7 @@ An example event for `application` looks as following: "type": "logs" }, "ecs": { - "version": "8.5.1" + "version": "8.11.0" }, "elastic_agent": { "id": "2d054344-10a6-40d9-90c1-ea017fecfda3", @@ -89,6 +89,10 @@ An example event for `application` looks as following: } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | @@ -97,12 +101,6 @@ An example event for `application` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data coming in at a regular interval or not. | keyword | -| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | | hadoop.application.allocated.mb | Total memory allocated to the application's running containers (Mb) | long | | hadoop.application.allocated.v_cores | The total number of virtual cores allocated to the application's running containers | long | | hadoop.application.id | Application ID | keyword | @@ -113,9 +111,7 @@ An example event for `application` looks as following: | hadoop.application.time.finished | Application finished time | date | | hadoop.application.time.started | Application start time | date | | hadoop.application.vcore_seconds | The amount of CPU resources the application has allocated | long | -| host.ip | Host ip addresses. | ip | | input.type | Type of Filebeat input. | keyword | -| tags | User defined tags. | keyword | ## cluster @@ -140,7 +136,7 @@ An example event for `cluster` looks as following: "type": "metrics" }, "ecs": { - "version": "8.5.1" + "version": "8.11.0" }, "elastic_agent": { "id": "abf8f8c1-f293-4e16-a8f8-8cf48014d040", @@ -186,7 +182,7 @@ An example event for `cluster` looks as following: "172.27.0.7" ], "mac": [ - "02:42:ac:1b:00:07" + "02-42-AC-1F-00-07" ], "name": "docker-fleet-agent", "os": { @@ -213,6 +209,10 @@ An example event for `cluster` looks as following: } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | Metric Type | @@ -228,12 +228,6 @@ An example event for `cluster` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | data_stream.type | Data stream type. | constant_keyword | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data coming in at a regular interval or not. | keyword | | -| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | | | hadoop.cluster.application_main.launch_delay_avg_time | Application Main Launch Delay Average Time (Milliseconds) | long | gauge | | hadoop.cluster.application_main.launch_delay_num_ops | Application Main Launch Delay Operations (Number of Operations) | long | gauge | | hadoop.cluster.application_main.register_delay_avg_time | Application Main Register Delay Average Time (Milliseconds) | long | gauge | @@ -268,11 +262,8 @@ An example event for `cluster` looks as following: | hadoop.cluster.virtual_cores.available | The number of available virtual cores | long | gauge | | hadoop.cluster.virtual_cores.reserved | The number of reserved virtual cores | long | gauge | | hadoop.cluster.virtual_cores.total | The total number of virtual cores | long | gauge | -| host.ip | Host ip addresses. | ip | | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | +| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | | service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | -| tags | List of keywords used to tag each event. | keyword | | ## datanode @@ -297,7 +288,7 @@ An example event for `datanode` looks as following: "type": "metrics" }, "ecs": { - "version": "8.5.1" + "version": "8.11.0" }, "elastic_agent": { "id": "2d054344-10a6-40d9-90c1-ea017fecfda3", @@ -379,6 +370,10 @@ An example event for `datanode` looks as following: } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | Metric Type | @@ -394,12 +389,6 @@ An example event for `datanode` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | data_stream.type | Data stream type. | constant_keyword | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data coming in at a regular interval or not. | keyword | | -| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | | | hadoop.datanode.blocks.cached | The number of blocks cached | long | gauge | | hadoop.datanode.blocks.failed.to_cache | The number of blocks that failed to cache | long | gauge | | hadoop.datanode.blocks.failed.to_uncache | The number of failed blocks to remove from cache | long | gauge | @@ -413,11 +402,8 @@ An example event for `datanode` looks as following: | hadoop.datanode.estimated_capacity_lost_total | The estimated capacity lost in bytes | long | gauge | | hadoop.datanode.last_volume_failure_date | The date/time of the last volume failure in milliseconds since epoch | date | | | hadoop.datanode.volumes.failed | Number of failed volumes | long | gauge | -| host.ip | Host ip addresses. | ip | | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | +| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | | service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | -| tags | List of keywords used to tag each event. | keyword | | ## namenode @@ -442,7 +428,7 @@ An example event for `namenode` looks as following: "type": "metrics" }, "ecs": { - "version": "8.5.1" + "version": "8.11.0" }, "elastic_agent": { "id": "adf6847a-3726-4fe6-a202-147021ff3cbc", @@ -503,7 +489,7 @@ An example event for `namenode` looks as following: "192.168.160.7" ], "mac": [ - "02:42:c0:a8:a0:07" + "02-42-AC-1F-00-07" ], "name": "docker-fleet-agent", "os": { @@ -530,6 +516,10 @@ An example event for `namenode` looks as following: } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | Metric Type | @@ -545,12 +535,6 @@ An example event for `namenode` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | data_stream.type | Data stream type. | constant_keyword | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data coming in at a regular interval or not. | keyword | | -| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | | | hadoop.namenode.blocks.corrupt | Current number of blocks with corrupt replicas. | long | gauge | | hadoop.namenode.blocks.missing_repl_one | Current number of missing blocks with replication factor 1 | long | gauge | | hadoop.namenode.blocks.pending_deletion | Current number of blocks pending deletion | long | gauge | @@ -573,11 +557,8 @@ An example event for `namenode` looks as following: | hadoop.namenode.stale_data_nodes | Current number of DataNodes marked stale due to delayed heartbeat | long | gauge | | hadoop.namenode.total_load | Current number of connections | long | gauge | | hadoop.namenode.volume_failures_total | Total number of volume failures across all Datanodes | long | gauge | -| host.ip | Host ip addresses. | ip | | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | +| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | | service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | -| tags | List of keywords used to tag each event. | keyword | | ## node_manager @@ -601,7 +582,7 @@ An example event for `node_manager` looks as following: "type": "metrics" }, "ecs": { - "version": "8.5.1" + "version": "8.11.0" }, "elastic_agent": { "id": "adf6847a-3726-4fe6-a202-147021ff3cbc", @@ -645,7 +626,7 @@ An example event for `node_manager` looks as following: "192.168.160.7" ], "mac": [ - "02:42:c0:a8:a0:07" + "02-42-AC-1F-00-07" ], "name": "docker-fleet-agent", "os": { @@ -672,6 +653,10 @@ An example event for `node_manager` looks as following: } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | Metric Type | @@ -687,12 +672,6 @@ An example event for `node_manager` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | data_stream.type | Data stream type. | constant_keyword | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data coming in at a regular interval or not. | keyword | | -| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | | | hadoop.node_manager.allocated_containers | Containers Allocated | long | gauge | | hadoop.node_manager.container_launch_duration_avg_time | Container Launch Duration Average Time (Seconds) | long | gauge | | hadoop.node_manager.container_launch_duration_num_ops | Container Launch Duration Operations (Operations) | long | counter | @@ -702,9 +681,6 @@ An example event for `node_manager` looks as following: | hadoop.node_manager.containers.killed | Containers Killed | long | counter | | hadoop.node_manager.containers.launched | Containers Launched | long | counter | | hadoop.node_manager.containers.running | Containers Running | long | gauge | -| host.ip | Host ip addresses. | ip | | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | +| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | | service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | -| tags | List of keywords used to tag each event. | keyword | | diff --git a/packages/hadoop/manifest.yml b/packages/hadoop/manifest.yml index 95d9a39ed6a..5f06d6d35af 100644 --- a/packages/hadoop/manifest.yml +++ b/packages/hadoop/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.0" name: hadoop title: Hadoop -version: "1.6.0" +version: "1.7.0" description: Collect metrics from Apache Hadoop with Elastic Agent. type: integration categories: @@ -10,7 +10,7 @@ categories: - big_data conditions: kibana: - version: "^8.10.2" + version: "^8.13.0" elastic: subscription: basic screenshots: