[sophos] Allow user-defined timezone override mappings (elastic#11873)

Add a user option to set time zone mappings to standard IANA time zone IDs. Sophos XG can use non-standard, potentially ambiguous, time zones which are not supported by Java timezone handling. To resolve this problem, this adds a "Timezone Map" option which users can use to map a Sophos time zone to a standard IANA time zone supported by Java.
taylor-swanson · Jan 15, 2025 · 8ba1803 · 8ba1803
1 parent 8d6ac89
commit 8ba1803
Show file tree

Hide file tree

Showing 32 changed files with 709 additions and 275 deletions.
diff --git a/packages/sophos/_dev/build/docs/README.md b/packages/sophos/_dev/build/docs/README.md
@@ -32,6 +32,14 @@ This is the Sophos `xg` dataset. Reference information about the log formats
 can be found in the [Sophos syslog guide](
 https://docs.sophos.com/nsg/sophos-firewall/18.5/PDF/SF%20syslog%20guide%2018.5.pdf).
 
+#### Timezones
+
+The format of timezones added to Sophos XG logs do not always match the expected formats used in common programming languages, and therefore 2 options have been added to the integration configuration:
+
+1. `Timezone` - This option allows the user to specify the timezone that the logs will be translated to. This will enforce all logs sent to the integration to the same timezone. This option is recommended for most users and default is `UTC`.
+
+2. `Timezone Map` - This option is for users who have logs from multiple timezones and want to translate them to the correct timezone. This option allows the user to specify a map of timezones to translate from and to. This option is recommended for advanced users who have logs from multiple timezones being sent to the same integration instance.
+
 {{event "xg"}}
 
 {{fields "xg"}}
diff --git a/packages/sophos/changelog.yml b/packages/sophos/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "3.10.0"
+  changes:
+    - description: Support user-defined timezones
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/11873
 - version: "3.9.2"
   changes:
     - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines.

diff --git a/packages/sophos/data_stream/utm/_dev/test/pipeline/test-sophos-utm-dhcp.log-expected.json b/packages/sophos/data_stream/utm/_dev/test/pipeline/test-sophos-utm-dhcp.log-expected.json
@@ -256,4 +256,4 @@
             ]
         }
     ]
-}
+}
diff --git a/packages/sophos/data_stream/utm/_dev/test/pipeline/test-sophos-utm-dns.log-expected.json b/packages/sophos/data_stream/utm/_dev/test/pipeline/test-sophos-utm-dns.log-expected.json
@@ -287,4 +287,4 @@
             ]
         }
     ]
-}
+}
diff --git a/packages/sophos/data_stream/utm/_dev/test/pipeline/test-sophos-utm-http.log-expected.json b/packages/sophos/data_stream/utm/_dev/test/pipeline/test-sophos-utm-http.log-expected.json
@@ -1314,4 +1314,4 @@
             }
         }
     ]
-}
+}
diff --git a/.../sophos/data_stream/utm/_dev/test/pipeline/test-sophos-utm-packetfilter.log-expected.json b/.../sophos/data_stream/utm/_dev/test/pipeline/test-sophos-utm-packetfilter.log-expected.json
@@ -349,4 +349,4 @@
             ]
         }
     ]
-}
+}
diff --git a/packages/sophos/data_stream/xg/_dev/test/pipeline/test-common-config.yml b/packages/sophos/data_stream/xg/_dev/test/pipeline/test-common-config.yml
@@ -8,6 +8,13 @@ fields:
   tags:
     - preserve_original_event
   _conf:
+    tz_map:
+      - tz_short: IST
+        tz_long: Asia/Kolkata
+      - tz_short: CEST
+        tz_long: Etc/GMT-2
+      - tz_short: BST
+        tz_long: Etc/GMT-1
     default: defaulttest.local
     mappings:
       - serial_number: "1234567890123456"

diff --git a/...ges/sophos/data_stream/xg/_dev/test/pipeline/test-sophos-18-5-anti-spam.log-expected.json b/...ges/sophos/data_stream/xg/_dev/test/pipeline/test-sophos-18-5-anti-spam.log-expected.json
@@ -1,7 +1,7 @@
 {
     "expected": [
         {
-            "@timestamp": "2017-01-31T18:28:25.000Z",
+            "@timestamp": "2017-01-31T18:28:25.000+05:30",
             "destination": {
                 "bytes": 0,
                 "ip": "10.198.233.61",
@@ -37,6 +37,7 @@
                 "original": "device=\"SFW\" date=2017-01-31 time=18:28:25 timezone=\"IST\" device_name=\"CR750iNG-XP\" device_id=C44313350024-P29PUA log_id=041107413001 log_type=\"Anti-Spam\" log_component=\"SMTP\" log_subtype=\"Spam\" status=\"\" priority=Warning fw_rule_id=0 user_name=\"gaurav\" av_policy_name=\"Gaurav235\" from_email_address=\"[email protected]\" to_email_address=\"[email protected]\" email_subject=\"RPD Spam Test: Spam\" mailid=\"c000000b-1485867502\" mailsize=400 spamaction=\"DROP\" reason=\"\" src_domainname=\"iview.com\" dst_domainname=\"\" src_ip=10.198.47.71 src_country_code=R1 dst_ip=10.198.233.61 dst_country_code=R1 protocol=\"TCP\" src_port=22258 dst_port=25 sent_bytes=0 recv_bytes=0 quarantine_reason=\"Spam\"",
                 "outcome": "success",
                 "severity": 4,
+                "timezone": "Asia/Kolkata",
                 "type": [
                     "info",
                     "denied",
@@ -107,7 +108,7 @@
             ]
         },
         {
-            "@timestamp": "2018-06-06T10:41:29.000Z",
+            "@timestamp": "2018-06-06T10:41:29.000+05:30",
             "destination": {
                 "bytes": 0,
                 "ip": "10.198.16.204",
@@ -144,6 +145,7 @@
                 "outcome": "success",
                 "reason": "Mail detected as PROBABLE SPAM.",
                 "severity": 4,
+                "timezone": "Asia/Kolkata",
                 "type": [
                     "info",
                     "denied",
@@ -213,7 +215,7 @@
             ]
         },
         {
-            "@timestamp": "2017-01-31T18:36:22.000Z",
+            "@timestamp": "2017-01-31T18:36:22.000+05:30",
             "destination": {
                 "bytes": 0,
                 "ip": "10.198.233.61",
@@ -248,6 +250,7 @@
                 "original": "device=\"SFW\" date=2017-01-31 time=18:36:22 timezone=\"IST\" device_name=\"CR750iNG-XP\" device_id=C44313350024-P29PUA log_id=041105613003 log_type=\"Anti-Spam\" log_component=\"SMTP\" log_subtype=\"Clean\" status=\"\" priority=Information fw_rule_id=0 user_name=\"gaurav\" av_policy_name=\"None\" from_email_address=\"[email protected]\" to_email_address=\"[email protected]\" email_subject=\"EMAIL\" mailid=\"<[email protected]>\" mailsize=398 spamaction=\"Accept\" reason=\"\" src_domainname=\"iview.com\" dst_domainname=\"\" src_ip=10.198.47.71 src_country_code=R1 dst_ip=10.198.233.61 dst_country_code=R1 protocol=\"TCP\" src_port=22477 dst_port=25 sent_bytes=0 recv_bytes=0 quarantine_reason=\"Other\"",
                 "outcome": "success",
                 "severity": 6,
+                "timezone": "Asia/Kolkata",
                 "type": [
                     "allowed",
                     "connection"
@@ -317,7 +320,7 @@
             ]
         },
         {
-            "@timestamp": "2018-06-06T11:08:08.000Z",
+            "@timestamp": "2018-06-06T11:08:08.000+05:30",
             "destination": {
                 "bytes": 0,
                 "ip": "10.198.17.121",
@@ -354,6 +357,7 @@
                 "outcome": "success",
                 "reason": "Sender IP address is blacklisted.",
                 "severity": 4,
+                "timezone": "Asia/Kolkata",
                 "type": [
                     "info",
                     "denied",
@@ -423,7 +427,7 @@
             ]
         },
         {
-            "@timestamp": "2017-01-31T18:34:41.000Z",
+            "@timestamp": "2017-01-31T18:34:41.000+05:30",
             "destination": {
                 "bytes": 0,
                 "ip": "10.198.233.61",
@@ -459,6 +463,7 @@
                 "original": "device=\"SFW\" date=2017-01-31 time=18:34:41 timezone=\"IST\" device_name=\"CR750iNG-XP\" device_id=C44313350024-P29PUA log_id=041113413005 log_type=\"Anti-Spam\" log_component=\"SMTP\" log_subtype=\"Outbound Spam\" status=\"\" priority=Warning fw_rule_id=0 user_name=\"gaurav\" av_policy_name=\"Gaurav123\" from_email_address=\"[email protected]\" to_email_address=\"[email protected]\" email_subject=\"RPD Spam Test: Spam\" mailid=\"<[email protected]>\" mailsize=405 spamaction=\"Accept\" reason=\"\" src_domainname=\"iview.com\" dst_domainname=\"\" src_ip=10.198.47.71 src_country_code=R1 dst_ip=10.198.233.61 dst_country_code=R1 protocol=\"TCP\" src_port=22420 dst_port=25 sent_bytes=0 recv_bytes=0 quarantine_reason=\"Spam\"",
                 "outcome": "success",
                 "severity": 4,
+                "timezone": "Asia/Kolkata",
                 "type": [
                     "info",
                     "denied",
@@ -529,7 +534,7 @@
             ]
         },
         {
-            "@timestamp": "2018-06-06T11:10:11.000Z",
+            "@timestamp": "2018-06-06T11:10:11.000+05:30",
             "destination": {
                 "bytes": 0,
                 "ip": "10.198.234.240",
@@ -566,6 +571,7 @@
                 "outcome": "success",
                 "reason": "Mail detected as OUTBOUND PROBABLE SPAM.",
                 "severity": 4,
+                "timezone": "Asia/Kolkata",
                 "type": [
                     "info",
                     "denied",
@@ -635,7 +641,7 @@
             ]
         },
         {
-            "@timestamp": "2018-06-06T12:50:07.000Z",
+            "@timestamp": "2018-06-06T12:50:07.000+05:30",
             "destination": {
                 "bytes": 0,
                 "ip": "10.198.17.121",
@@ -672,6 +678,7 @@
                 "outcome": "success",
                 "reason": "Email containing confidential data detected. Relevant Data Protection Policy applied.",
                 "severity": 6,
+                "timezone": "Asia/Kolkata",
                 "type": [
                     "info",
                     "denied",
@@ -741,7 +748,7 @@
             ]
         },
         {
-            "@timestamp": "2018-06-06T12:51:34.000Z",
+            "@timestamp": "2018-06-06T12:51:34.000+05:30",
             "destination": {
                 "bytes": 0,
                 "ip": "10.198.16.204",
@@ -777,6 +784,7 @@
                 "outcome": "success",
                 "reason": "SPX Template of type Specified by Sender successfully applied on Email.",
                 "severity": 6,
+                "timezone": "Asia/Kolkata",
                 "type": [
                     "allowed",
                     "connection"
@@ -845,7 +853,7 @@
             ]
         },
         {
-            "@timestamp": "2018-06-06T12:52:49.000Z",
+            "@timestamp": "2018-06-06T12:52:49.000+05:30",
             "destination": {
                 "bytes": 0,
                 "ip": "10.198.16.204",
@@ -880,7 +888,8 @@
                 "original": "device=\"SFW\" date=2018-06-06 time=12:52:49 timezone=\"IST\" device_name=\"SG430\" device_id=S4000806149EE49 log_id=041122613011 log_type=\"Anti-Spam\" log_component=\"SMTP\" log_subtype=\"SPX\" status=\"\" priority=Information fw_rule_id=0 user_name=\"\" av_policy_name=\"None\" from_email_address=\"[email protected]\" to_email_address=\"[email protected]\" email_subject=\"Test failed\" mailid=\"c0000004-1528269769\" mailsize=431 spamaction=\"REJECT\" reason=\"Email could not be SPX- encrypted because password was not found in the Email subject.\" src_domainname=\"postman.local\" dst_domainname=\"\" src_ip=10.198.16.121 src_country_code=R1 dst_ip=10.198.16.204 dst_country_code=R1 protocol=\"TCP\" src_port=60305 dst_port=25 sent_bytes=0 recv_bytes=0 quarantine_reason=\"Other\"",
                 "outcome": "success",
                 "reason": "Email could not be SPX- encrypted because password was not found in the Email subject.",
-                "severity": 6
+                "severity": 6,
+                "timezone": "Asia/Kolkata"
             },
             "host": {
                 "name": "defaulttest.local"
@@ -945,7 +954,7 @@
             ]
         },
         {
-            "@timestamp": "2018-06-06T12:53:39.000Z",
+            "@timestamp": "2018-06-06T12:53:39.000+05:30",
             "destination": {
                 "bytes": 0,
                 "ip": "10.198.17.121",
@@ -966,6 +975,7 @@
                 "outcome": "success",
                 "reason": "SMTP DoS",
                 "severity": 4,
+                "timezone": "Asia/Kolkata",
                 "type": [
                     "info",
                     "denied",
@@ -1028,7 +1038,7 @@
             ]
         },
         {
-            "@timestamp": "2017-01-31T15:46:45.000Z",
+            "@timestamp": "2017-01-31T15:46:45.000+05:30",
             "destination": {
                 "bytes": 0,
                 "ip": "10.198.233.61",
@@ -1064,6 +1074,7 @@
                 "outcome": "success",
                 "reason": "Mail is marked Clean by Sophos Sandstorm.",
                 "severity": 6,
+                "timezone": "Asia/Kolkata",
                 "type": [
                     "allowed",
                     "connection"
@@ -1133,7 +1144,7 @@
             ]
         },
         {
-            "@timestamp": "2018-06-06T12:56:53.000Z",
+            "@timestamp": "2018-06-06T12:56:53.000+05:30",
             "destination": {
                 "bytes": 0,
                 "ip": "10.198.17.121",
@@ -1170,6 +1181,7 @@
                 "outcome": "success",
                 "reason": "Email is marked Malicious by Sophos Sandstorm.",
                 "severity": 4,
+                "timezone": "Asia/Kolkata",
                 "type": [
                     "info",
                     "denied",
@@ -1238,7 +1250,7 @@
             ]
         },
         {
-            "@timestamp": "2017-01-31T18:31:11.000Z",
+            "@timestamp": "2017-01-31T18:31:11.000+05:30",
             "destination": {
                 "bytes": 0,
                 "ip": "10.198.233.61",
@@ -1274,6 +1286,7 @@
                 "original": "device=\"SFW\" date=2017-01-31 time=18:31:11 timezone=\"IST\" device_name=\"CR750iNG-XP\" device_id=C44313350024-P29PUA log_id=041207414001 log_type=\"Anti-Spam\" log_component=\"POP3\" log_subtype=\"Spam\" status=\"\" priority=Warning fw_rule_id=0 user_name=\"gaurav\" av_policy_name=\"GauravPatel\" from_email_address=\"[email protected]\" to_email_address=\"gaurav2@iview. com\" email_subject=\"RPD Spam Test: Spam\" mailid=\"<[email protected]>\" mailsize=574 spamaction=\"Accept\" reason=\"\" src_domainname=\"iview.com\" dst_domainname=\"iview.com\" src_ip=10.198.47.71 src_country_code=R1 dst_ip=10.198.233.61 dst_country_code=R1 protocol=\"TCP\" src_port=22333 dst_port=110 sent_bytes=0 recv_bytes=0 quarantine_reason=\"Other\"",
                 "outcome": "success",
                 "severity": 4,
+                "timezone": "Asia/Kolkata",
                 "type": [
                     "info",
                     "denied",
@@ -1345,7 +1358,7 @@
             ]
         },
         {
-            "@timestamp": "2018-06-06T12:59:01.000Z",
+            "@timestamp": "2018-06-06T12:59:01.000+05:30",
             "destination": {
                 "bytes": 0,
                 "ip": "10.198.234.240",
@@ -1382,6 +1395,7 @@
                 "outcome": "success",
                 "reason": "Mail detected as PROBABLE SPAM",
                 "severity": 4,
+                "timezone": "Asia/Kolkata",
                 "type": [
                     "info",
                     "denied",
@@ -1449,7 +1463,7 @@
             ]
         },
         {
-            "@timestamp": "2018-06-06T13:00:34.000Z",
+            "@timestamp": "2018-06-06T13:00:34.000+05:30",
             "destination": {
                 "bytes": 0,
                 "ip": "10.198.234.240",
@@ -1485,6 +1499,7 @@
                 "outcome": "success",
                 "reason": "Mail is Clean",
                 "severity": 6,
+                "timezone": "Asia/Kolkata",
                 "type": [
                     "allowed",
                     "connection"
@@ -1551,7 +1566,7 @@
             ]
         },
         {
-            "@timestamp": "2018-06-06T13:01:42.000Z",
+            "@timestamp": "2018-06-06T13:01:42.000+05:30",
             "destination": {
                 "bytes": 0,
                 "ip": "10.198.234.240",
@@ -1588,6 +1603,7 @@
                 "outcome": "success",
                 "reason": "Mail detected as SPAM",
                 "severity": 4,
+                "timezone": "Asia/Kolkata",
                 "type": [
                     "info",
                     "denied",
@@ -1655,7 +1671,7 @@
             ]
         },
         {
-            "@timestamp": "2018-06-06T13:02:54.000Z",
+            "@timestamp": "2018-06-06T13:02:54.000+05:30",
             "destination": {
                 "bytes": 0,
                 "ip": "10.198.234.240",
@@ -1692,6 +1708,7 @@
                 "outcome": "success",
                 "reason": "Mail detected as PROBABLE SPAM",
                 "severity": 4,
+                "timezone": "Asia/Kolkata",
                 "type": [
                     "info",
                     "denied",
@@ -1759,7 +1776,7 @@
             ]
         },
         {
-            "@timestamp": "2018-06-06T13:03:58.000Z",
+            "@timestamp": "2018-06-06T13:03:58.000+05:30",
             "destination": {
                 "bytes": 0,
                 "ip": "10.198.234.240",
@@ -1795,6 +1812,7 @@
                 "outcome": "success",
                 "reason": "Mail is Clean",
                 "severity": 6,
+                "timezone": "Asia/Kolkata",
                 "type": [
                     "allowed",
                     "connection"
@@ -1861,7 +1879,7 @@
             ]
         },
         {
-            "@timestamp": "2018-06-05T19:11:26.000Z",
+            "@timestamp": "2018-06-05T19:11:26.000+05:30",
             "destination": {
                 "bytes": 0,
                 "ip": "10.198.16.204",
@@ -1897,6 +1915,7 @@
                 "outcome": "success",
                 "reason": "Email has been delivered to recipient(s).",
                 "severity": 6,
+                "timezone": "Asia/Kolkata",
                 "type": [
                     "allowed",
                     "connection"
@@ -1965,4 +1984,4 @@
             ]
         }
     ]
-}
+}
-Original file line number
+Diff line change
@@ Expand Up / @@ -256,4 +256,4 @@ @@
                 ]
             }
         ]
-    }
+    }