zscaler_zia.firewall: Fix source/destination ip mapping (elastic#11613)

[zscaler_zia] Fix source/destination ip mapping in `firewall` logs. Currently the ECS `source.ip` and `destination.ip` are copied not only from ZScaler client's source and destination ips, but also from the proxy server's source and destination ips. From security detection point of view, the flow is clearly defined if `source.ip` and `destination.ip` are mapped only from client's perspective instead of mixing them with proxy's source and destination ips. Also, the current array representation of `source.ip` and `destination.ip` makes it unable to be used with `geoip` processor. This PR: - Removes the mapping from proxy server's source and destination to ECS `source.ip` and `destination.ip` respectively. - Adds `source.nat.ip` from zscaler's `tsip` field. - Adds `geoip` processor to `source.ip` and `destination.ip`. - Updates `related.ip` to reflect from custom fields of proxy's source and destination ips.
agithomas · Nov 6, 2024 · a773b73 · a773b73
1 parent 90580c4
commit a773b73
Show file tree

Hide file tree

Showing 8 changed files with 156 additions and 204 deletions.
diff --git a/packages/zscaler_zia/changelog.yml b/packages/zscaler_zia/changelog.yml
@@ -1,4 +1,12 @@
 # newer versions go on top
+- version: "3.3.0"
+  changes:
+    - description: Fix source/destination ip mapping in firewall logs.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/11613
+    - description: Add geoip processor to source and destination ip.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/11613
 - version: "3.2.4"
   changes:
     - description: Improve data processing in the web pipeline.

diff --git a/...zia/data_stream/firewall/_dev/test/pipeline/test-firewall-http-endpoint.log-expected.json b/...zia/data_stream/firewall/_dev/test/pipeline/test-firewall-http-endpoint.log-expected.json
@@ -8,9 +8,7 @@
                 "geo": {
                     "country_iso_code": "USA"
                 },
-                "ip": [
-                    "1.128.0.0"
-                ],
+                "ip": "1.128.0.0",
                 "port": [
                     22,
                     443
@@ -87,10 +85,10 @@
                 "geo": {
                     "country_name": "United States"
                 },
-                "ip": [
-                    "0.0.0.0",
-                    "1.128.0.0"
-                ],
+                "ip": "0.0.0.0",
+                "nat": {
+                    "ip": "89.160.20.128"
+                },
                 "port": [
                     22
                 ]
@@ -238,9 +236,7 @@
             "@timestamp": "2022-12-31T02:22:22.000Z",
             "destination": {
                 "bytes": 0,
-                "ip": [
-                    "0.0.0.0"
-                ],
+                "ip": "0.0.0.0",
                 "port": [
                     120,
                     456
@@ -279,9 +275,10 @@
             },
             "source": {
                 "bytes": 0,
-                "ip": [
-                    "0.0.0.0"
-                ],
+                "ip": "0.0.0.0",
+                "nat": {
+                    "ip": "0.0.0.0"
+                },
                 "port": [
                     123,
                     0
@@ -348,12 +345,15 @@
                 "bytes": 10000,
                 "domain": "www.example.com",
                 "geo": {
-                    "country_iso_code": "USA"
+                    "continent_name": "Europe",
+                    "country_iso_code": "NO",
+                    "country_name": "Norway",
+                    "location": {
+                        "lat": 62.0,
+                        "lon": 10.0
+                    }
                 },
-                "ip": [
-                    "2a02:cf40::",
-                    "67.43.156.0"
-                ],
+                "ip": "2a02:cf40::",
                 "port": [
                     22,
                     443
@@ -407,8 +407,8 @@
                 ],
                 "ip": [
                     "2a02:cf40::",
-                    "67.43.156.0",
                     "0.0.0.0",
+                    "67.43.156.0",
                     "1.128.0.0",
                     "89.160.20.128"
                 ],
@@ -432,10 +432,10 @@
                 "geo": {
                     "country_name": "United States"
                 },
-                "ip": [
-                    "0.0.0.0",
-                    "1.128.0.0"
-                ],
+                "ip": "0.0.0.0",
+                "nat": {
+                    "ip": "89.160.20.128"
+                },
                 "port": [
                     25,
                     22

diff --git a/packages/zscaler_zia/data_stream/firewall/_dev/test/pipeline/test-firewall.log-expected.json b/packages/zscaler_zia/data_stream/firewall/_dev/test/pipeline/test-firewall.log-expected.json
@@ -8,9 +8,7 @@
                 "geo": {
                     "country_iso_code": "USA"
                 },
-                "ip": [
-                    "1.128.0.0"
-                ],
+                "ip": "1.128.0.0",
                 "port": [
                     22,
                     443
@@ -86,9 +84,10 @@
                 "geo": {
                     "country_name": "United States"
                 },
-                "ip": [
-                    "1.128.0.0"
-                ],
+                "ip": "1.128.0.0",
+                "nat": {
+                    "ip": "89.160.20.128"
+                },
                 "port": [
                     22
                 ]
@@ -238,12 +237,15 @@
                 "bytes": 10000,
                 "domain": "www.example.com",
                 "geo": {
-                    "country_iso_code": "USA"
+                    "continent_name": "Europe",
+                    "country_iso_code": "NO",
+                    "country_name": "Norway",
+                    "location": {
+                        "lat": 62.0,
+                        "lon": 10.0
+                    }
                 },
-                "ip": [
-                    "2a02:cf40::",
-                    "67.43.156.0"
-                ],
+                "ip": "2a02:cf40::",
                 "port": [
                     22,
                     443
@@ -297,8 +299,8 @@
                 ],
                 "ip": [
                     "2a02:cf40::",
-                    "67.43.156.0",
                     "0.0.0.0",
+                    "67.43.156.0",
                     "1.128.0.0",
                     "89.160.20.128"
                 ],
@@ -322,10 +324,10 @@
                 "geo": {
                     "country_name": "United States"
                 },
-                "ip": [
-                    "0.0.0.0",
-                    "1.128.0.0"
-                ],
+                "ip": "0.0.0.0",
+                "nat": {
+                    "ip": "89.160.20.128"
+                },
                 "port": [
                     25,
                     22

diff --git a/packages/zscaler_zia/data_stream/firewall/_dev/test/pipeline/test-unicode.json-expected.json b/packages/zscaler_zia/data_stream/firewall/_dev/test/pipeline/test-unicode.json-expected.json
@@ -6,12 +6,15 @@
                 "bytes": 10000,
                 "domain": "www.example.com",
                 "geo": {
-                    "country_iso_code": "USA"
+                    "continent_name": "Europe",
+                    "country_iso_code": "NO",
+                    "country_name": "Norway",
+                    "location": {
+                        "lat": 62.0,
+                        "lon": 10.0
+                    }
                 },
-                "ip": [
-                    "2a02:cf40::",
-                    "67.43.156.0"
-                ],
+                "ip": "2a02:cf40::",
                 "port": [
                     22,
                     443
@@ -65,8 +68,8 @@
                 ],
                 "ip": [
                     "2a02:cf40::",
-                    "67.43.156.0",
                     "0.0.0.0",
+                    "67.43.156.0",
                     "1.128.0.0",
                     "89.160.20.128"
                 ],
@@ -90,10 +93,10 @@
                 "geo": {
                     "country_name": "United States"
                 },
-                "ip": [
-                    "0.0.0.0",
-                    "1.128.0.0"
-                ],
+                "ip": "0.0.0.0",
+                "nat": {
+                    "ip": "89.160.20.128"
+                },
                 "port": [
                     25,
                     22

diff --git a/packages/zscaler_zia/data_stream/firewall/elasticsearch/ingest_pipeline/default.yml b/packages/zscaler_zia/data_stream/firewall/elasticsearch/ingest_pipeline/default.yml
@@ -164,12 +164,22 @@ processors:
         - append:
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
-  - append:
+  - set:
+      field: destination.ip
+      tag: set_destination_ip_from_zscaler_zia_firewall_client_destination_ip
+      copy_from: zscaler_zia.firewall.client.destination.ip
+      ignore_empty_value: true
+  - geoip:
       field: destination.ip
-      tag: append_zscaler_zia_firewall_client_destination_ip_into_destination_ip
-      value: '{{{zscaler_zia.firewall.client.destination.ip}}}'
+      target_field: destination.geo
+      tag: geoip_destination_ip
+      ignore_missing: true
+  - append:
+      field: related.ip
+      value: '{{{destination.ip}}}'
+      tag: append_related_ip_from_destination_ip
+      if: ctx.destination?.ip != null
       allow_duplicates: false
-      if: ctx.zscaler_zia?.firewall?.client?.destination?.ip != null
   - convert:
       field: json.cdport
       tag: convert_cdport_to_long
@@ -208,12 +218,22 @@ processors:
         - append:
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
-  - append:
+  - set:
+      field: source.ip
+      tag: set_source_ip_from_zscaler_zia_firewall_client_source_ip
+      copy_from: zscaler_zia.firewall.client.source.ip
+      ignore_empty_value: true
+  - geoip:
       field: source.ip
-      tag: append_zscaler_zia_firewall_client_source_ip_into_source_ip
-      value: '{{{zscaler_zia.firewall.client.source.ip}}}'
+      target_field: source.geo
+      tag: geoip_source_ip
+      ignore_missing: true
+  - append:
+      field: related.ip
+      value: '{{{source.ip}}}'
+      tag: append_related_ip_from_source_ip
+      if: ctx.source?.ip != null
       allow_duplicates: false
-      if: ctx.zscaler_zia?.firewall?.client?.source?.ip != null
   - convert:
       field: json.csport
       tag: convert_csport_to_long
@@ -283,6 +303,7 @@ processors:
       field: destination.geo.country_iso_code
       tag: set_destination_geo_country_iso_code_from_firewall_destination_country
       copy_from: zscaler_zia.firewall.destination.country
+      if: ctx.destination?.geo?.country_iso_code == null
       ignore_empty_value: true
   - rename:
       field: json.deviceappversion
@@ -763,21 +784,11 @@ processors:
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - append:
-      field: destination.ip
-      tag: append_zscaler_zia_firewall_server_destination_ip_into_destination_ip
+      field: related.ip
       value: '{{{zscaler_zia.firewall.server.destination.ip}}}'
-      allow_duplicates: false
+      tag: append_related_ip_from_zscaler_zia_firewall_server_destination_ip
       if: ctx.zscaler_zia?.firewall?.server?.destination?.ip != null
-  - foreach:
-      field: destination.ip
-      tag: foreach_destination_ip_to_append_related_ip_from_destination_ip
-      if: ctx.destination?.ip instanceof List
-      processor:
-        append:
-          field: related.ip
-          tag: append_related_ip_from_destination_ip
-          value: '{{{_ingest._value}}}'
-          allow_duplicates: false
+      allow_duplicates: false
   - convert:
       field: json.sdport
       tag: convert_sdport_to_long
@@ -821,21 +832,11 @@ processors:
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - append:
-      field: source.ip
-      tag: append_zscaler_zia_firewall_server_source_ip_into_source_ip
+      field: related.ip
       value: '{{{zscaler_zia.firewall.server.source.ip}}}'
-      allow_duplicates: false
+      tag: append_related_ip_from_zscaler_zia_firewall_server_source_ip
       if: ctx.zscaler_zia?.firewall?.server?.source?.ip != null
-  - foreach:
-      field: source.ip
-      tag: foreach_source_ip_to_append_related_ip_from_source_ip
-      if: ctx.source?.ip instanceof List
-      processor:
-        append:
-          field: related.ip
-          tag: append_related_ip_from_source_ip
-          value: '{{{_ingest._value}}}'
-          allow_duplicates: false
+      allow_duplicates: false
   - convert:
       field: json.ssport
       tag: convert_ssport_to_long
@@ -887,6 +888,7 @@ processors:
       field: source.geo.country_name
       tag: set_source_geo_country_name_from_firewall_source_ip_country
       copy_from: zscaler_zia.firewall.source_ip_country
+      if: ctx.source?.geo?.country_name == null
       ignore_empty_value: true
   - rename:
       field: json.stateful
@@ -969,6 +971,11 @@ processors:
         - append:
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
+  - set:
+      field: source.nat.ip
+      tag: set_source_nat_ip_from_zscaler_zia_firewall_tunnel_ip
+      copy_from: zscaler_zia.firewall.tunnel.ip
+      ignore_empty_value: true
   - append:
       field: related.ip
       value: '{{{zscaler_zia.firewall.tunnel.ip}}}'