From ca6ba575fbfe0ebe2a9dcb877d40737570f38e83 Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Tue, 26 Dec 2023 17:58:31 +0530 Subject: [PATCH] initial changes to salesforce module filesets --- .../module/salesforce/_meta/config.yml | 161 ++++------ .../salesforce/apex-rest/config/apex-rest.yml | 71 ----- .../module/salesforce/apex-rest/manifest.yml | 20 -- .../{apex-rest => apex}/_meta/fields.yml | 0 .../module/salesforce/apex/config/apex.yml | 79 +++++ .../{apex-rest => apex}/ingest/pipeline.yml | 0 .../module/salesforce/apex/manifest.yml | 24 ++ .../test/salesforce_apex_callout.ndjson.log | 0 ...orce_apex_callout.ndjson.log-expected.json | 0 .../test/salesforce_apex_execution.ndjson.log | 0 ...ce_apex_execution.ndjson.log-expected.json | 0 ...alesforce_apex_external_callout.ndjson.log | 0 ..._external_callout.ndjson.log-expected.json | 0 .../test/salesforce_apex_rest.ndjson.log | 0 ...esforce_apex_rest.ndjson.log-expected.json | 0 .../test/salesforce_apex_soap.ndjson.log | 0 ...esforce_apex_soap.ndjson.log-expected.json | 0 .../test/salesforce_apex_trigger.ndjson.log | 0 ...orce_apex_trigger.ndjson.log-expected.json | 0 .../login-rest/config/login-rest.yml | 71 ----- .../module/salesforce/login-rest/manifest.yml | 20 -- .../salesforce/login-stream/_meta/fields.yml | 58 ---- .../login-stream/config/login-stream.yml | 44 --- .../login-stream/ingest/pipeline.yml | 278 ------------------ .../salesforce/login-stream/manifest.yml | 16 - .../test/salesforce_login.ndjson.log | 1 - .../salesforce_login.ndjson.log-expected.json | 55 ---- .../{login-rest => login}/_meta/fields.yml | 0 .../module/salesforce/login/config/login.yml | 95 ++++++ .../{login-rest => login}/ingest/pipeline.yml | 0 .../module/salesforce/login/manifest.yml | 28 ++ .../test/salesforce_login.ndjson.log | 0 .../salesforce_login.ndjson.log-expected.json | 0 .../logout-rest/config/logout-rest.yml | 70 ----- .../salesforce/logout-rest/manifest.yml | 19 -- .../salesforce/logout-stream/_meta/fields.yml | 22 -- .../logout-stream/config/logout-stream.yml | 44 --- .../logout-stream/ingest/pipeline.yml | 147 --------- .../salesforce/logout-stream/manifest.yml | 15 - .../test/salesforce_logout.ndjson.log | 1 - ...salesforce_logout.ndjson.log-expected.json | 30 -- .../{logout-rest => logout}/_meta/fields.yml | 0 .../salesforce/logout/config/logout.yml | 95 ++++++ .../ingest/pipeline.yml | 0 .../module/salesforce/logout/manifest.yml | 27 ++ .../test/salesforce_logout.ndjson.log | 0 ...salesforce_logout.ndjson.log-expected.json | 0 .../config/setupaudittrail-rest.yml | 68 ----- .../setupaudittrail-rest/manifest.yml | 19 -- .../_meta/fields.yml | 0 .../config/setupaudittrail.yml | 83 ++++++ .../ingest/pipeline.yml | 0 .../salesforce/setupaudittrail/manifest.yml | 24 ++ .../salesforce_setupaudittrail.ndjson.log | 0 ...e_setupaudittrail.ndjson.log-expected.json | 0 55 files changed, 514 insertions(+), 1171 deletions(-) delete mode 100644 x-pack/filebeat/module/salesforce/apex-rest/config/apex-rest.yml delete mode 100644 x-pack/filebeat/module/salesforce/apex-rest/manifest.yml rename x-pack/filebeat/module/salesforce/{apex-rest => apex}/_meta/fields.yml (100%) create mode 100644 x-pack/filebeat/module/salesforce/apex/config/apex.yml rename x-pack/filebeat/module/salesforce/{apex-rest => apex}/ingest/pipeline.yml (100%) create mode 100644 x-pack/filebeat/module/salesforce/apex/manifest.yml rename x-pack/filebeat/module/salesforce/{apex-rest => apex}/test/salesforce_apex_callout.ndjson.log (100%) rename x-pack/filebeat/module/salesforce/{apex-rest => apex}/test/salesforce_apex_callout.ndjson.log-expected.json (100%) rename x-pack/filebeat/module/salesforce/{apex-rest => apex}/test/salesforce_apex_execution.ndjson.log (100%) rename x-pack/filebeat/module/salesforce/{apex-rest => apex}/test/salesforce_apex_execution.ndjson.log-expected.json (100%) rename x-pack/filebeat/module/salesforce/{apex-rest => apex}/test/salesforce_apex_external_callout.ndjson.log (100%) rename x-pack/filebeat/module/salesforce/{apex-rest => apex}/test/salesforce_apex_external_callout.ndjson.log-expected.json (100%) rename x-pack/filebeat/module/salesforce/{apex-rest => apex}/test/salesforce_apex_rest.ndjson.log (100%) rename x-pack/filebeat/module/salesforce/{apex-rest => apex}/test/salesforce_apex_rest.ndjson.log-expected.json (100%) rename x-pack/filebeat/module/salesforce/{apex-rest => apex}/test/salesforce_apex_soap.ndjson.log (100%) rename x-pack/filebeat/module/salesforce/{apex-rest => apex}/test/salesforce_apex_soap.ndjson.log-expected.json (100%) rename x-pack/filebeat/module/salesforce/{apex-rest => apex}/test/salesforce_apex_trigger.ndjson.log (100%) rename x-pack/filebeat/module/salesforce/{apex-rest => apex}/test/salesforce_apex_trigger.ndjson.log-expected.json (100%) delete mode 100644 x-pack/filebeat/module/salesforce/login-rest/config/login-rest.yml delete mode 100644 x-pack/filebeat/module/salesforce/login-rest/manifest.yml delete mode 100644 x-pack/filebeat/module/salesforce/login-stream/_meta/fields.yml delete mode 100644 x-pack/filebeat/module/salesforce/login-stream/config/login-stream.yml delete mode 100644 x-pack/filebeat/module/salesforce/login-stream/ingest/pipeline.yml delete mode 100644 x-pack/filebeat/module/salesforce/login-stream/manifest.yml delete mode 100644 x-pack/filebeat/module/salesforce/login-stream/test/salesforce_login.ndjson.log delete mode 100644 x-pack/filebeat/module/salesforce/login-stream/test/salesforce_login.ndjson.log-expected.json rename x-pack/filebeat/module/salesforce/{login-rest => login}/_meta/fields.yml (100%) create mode 100644 x-pack/filebeat/module/salesforce/login/config/login.yml rename x-pack/filebeat/module/salesforce/{login-rest => login}/ingest/pipeline.yml (100%) create mode 100644 x-pack/filebeat/module/salesforce/login/manifest.yml rename x-pack/filebeat/module/salesforce/{login-rest => login}/test/salesforce_login.ndjson.log (100%) rename x-pack/filebeat/module/salesforce/{login-rest => login}/test/salesforce_login.ndjson.log-expected.json (100%) delete mode 100644 x-pack/filebeat/module/salesforce/logout-rest/config/logout-rest.yml delete mode 100644 x-pack/filebeat/module/salesforce/logout-rest/manifest.yml delete mode 100644 x-pack/filebeat/module/salesforce/logout-stream/_meta/fields.yml delete mode 100644 x-pack/filebeat/module/salesforce/logout-stream/config/logout-stream.yml delete mode 100644 x-pack/filebeat/module/salesforce/logout-stream/ingest/pipeline.yml delete mode 100644 x-pack/filebeat/module/salesforce/logout-stream/manifest.yml delete mode 100644 x-pack/filebeat/module/salesforce/logout-stream/test/salesforce_logout.ndjson.log delete mode 100644 x-pack/filebeat/module/salesforce/logout-stream/test/salesforce_logout.ndjson.log-expected.json rename x-pack/filebeat/module/salesforce/{logout-rest => logout}/_meta/fields.yml (100%) create mode 100644 x-pack/filebeat/module/salesforce/logout/config/logout.yml rename x-pack/filebeat/module/salesforce/{logout-rest => logout}/ingest/pipeline.yml (100%) create mode 100644 x-pack/filebeat/module/salesforce/logout/manifest.yml rename x-pack/filebeat/module/salesforce/{logout-rest => logout}/test/salesforce_logout.ndjson.log (100%) rename x-pack/filebeat/module/salesforce/{logout-rest => logout}/test/salesforce_logout.ndjson.log-expected.json (100%) delete mode 100644 x-pack/filebeat/module/salesforce/setupaudittrail-rest/config/setupaudittrail-rest.yml delete mode 100644 x-pack/filebeat/module/salesforce/setupaudittrail-rest/manifest.yml rename x-pack/filebeat/module/salesforce/{setupaudittrail-rest => setupaudittrail}/_meta/fields.yml (100%) create mode 100644 x-pack/filebeat/module/salesforce/setupaudittrail/config/setupaudittrail.yml rename x-pack/filebeat/module/salesforce/{setupaudittrail-rest => setupaudittrail}/ingest/pipeline.yml (100%) create mode 100644 x-pack/filebeat/module/salesforce/setupaudittrail/manifest.yml rename x-pack/filebeat/module/salesforce/{setupaudittrail-rest => setupaudittrail}/test/salesforce_setupaudittrail.ndjson.log (100%) rename x-pack/filebeat/module/salesforce/{setupaudittrail-rest => setupaudittrail}/test/salesforce_setupaudittrail.ndjson.log-expected.json (100%) diff --git a/x-pack/filebeat/module/salesforce/_meta/config.yml b/x-pack/filebeat/module/salesforce/_meta/config.yml index f5c6c9c7f069..3dc40f902aeb 100644 --- a/x-pack/filebeat/module/salesforce/_meta/config.yml +++ b/x-pack/filebeat/module/salesforce/_meta/config.yml @@ -1,130 +1,87 @@ - module: salesforce - apex-rest: + apex: enabled: false + var.api_version: 56 - # Oauth Client ID - #var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + var.jwt_enabled: true - # Oauth Client Secret - #var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + var.client_username: "abc.xyz@mail.com" + var.client_key_path: client_key.pem + var.jwt_url: https://login.salesforce.com - # Oauth Token URL - #var.token_url: "https://login.salesforce.com/services/oauth2/token" + var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + var.token_url: "https://login.salesforce.com" + var.user: "abc.xyz@mail.com" + var.password: "P@$$W0₹D" - # Oauth User, should include the User mail - #var.user: "abc.xyz@mail.com" - - # Oauth password, should include the User password - #var.password: "P@$$W0₹D" - - # URL, should include the instance_url - #var.url: "https://instance_id.my.salesforce.com" + var.url: "https://instance_id.my.salesforce.com" + var.elf_interval: 1h - login-rest: - enabled: false - - # Oauth Client ID - #var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - - # Oauth Client Secret - #var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - - # Oauth Token URL - #var.token_url: "https://login.salesforce.com/services/oauth2/token" - - # Oauth User, should include the User mail - #var.user: "abc.xyz@mail.com" - - # Oauth password, should include the User password - #var.password: "P@$$W0₹D" - - # URL, should include the instance_url - #var.url: "https://instance_id.my.salesforce.com" - - login-stream: - enabled: false - - # Oauth Client ID - #var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - - # Oauth Client Secret - #var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - - # Oauth Token URL - #var.token_url: "https://login.salesforce.com/services/oauth2/token" - - # Oauth User, should include the User mail - #var.user: "abc.xyz@mail.com" - - # Oauth password, should include the User password - #var.password: "P@$$W0₹D" - - # URL, should include the instance_url - #var.url: "https://instance_id.my.salesforce.com" - - logout-rest: + login: enabled: false + var.api_version: 56 - # Oauth Client ID - #var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - - # Oauth Client Secret - #var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + var.jwt_enabled: true - # Oauth Token URL - #var.token_url: "https://login.salesforce.com/services/oauth2/token" + var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + var.client_username: "abc.xyz@mail.com" + var.client_key_path: client_key.pem + var.jwt_url: https://login.salesforce.com - # Oauth User, should include the User mail - #var.user: "abc.xyz@mail.com" + var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + var.token_url: "https://login.salesforce.com" + var.user: "abc.xyz@mail.com" + var.password: "P@$$W0₹D" - # Oauth password, should include the User password - #var.password: "P@$$W0₹D" + var.url: "https://instance_id.my.salesforce.com" - # URL, should include the instance_url - #var.url: "https://instance_id.my.salesforce.com" + var.event_log_file: true + var.elf_interval: 1h + var.real_time: true + var.real_time_interval: 5m - logout-stream: + logout: enabled: false + var.api_version: 56 - # Oauth Client ID - #var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - - # Oauth Client Secret - #var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + var.jwt_enabled: true - # Oauth Token URL - #var.token_url: "https://login.salesforce.com/services/oauth2/token" + var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + var.client_username: "abc.xyz@mail.com" + var.client_key_path: client_key.pem + var.jwt_url: https://login.salesforce.com - # Oauth User, should include the User mail - #var.user: "abc.xyz@mail.com" + var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + var.token_url: "https://login.salesforce.com" + var.user: "abc.xyz@mail.com" + var.password: "P@$$W0₹D" - # Oauth password, should include the User password - #var.password: "P@$$W0₹D" + var.url: "https://instance_id.my.salesforce.com" - # URL, should include the instance_url - #var.url: "https://instance_id.my.salesforce.com" + var.event_log_file: true + var.elf_interval: 1h + var.real_time: true + var.real_time_interval: 5m - setupaudittrail-rest: + setupaudittrail: enabled: false + var.api_version: 56 - # Oauth Client ID - #var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - - # Oauth Client Secret - #var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - - # Oauth Token URL - #var.token_url: "https://login.salesforce.com/services/oauth2/token" + var.jwt_enabled: true - # Oauth User, should include the User mail - #var.user: "abc.xyz@mail.com" + var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + var.client_username: "abc.xyz@mail.com" + var.client_key_path: client_key.pem + var.jwt_url: https://login.salesforce.com - # Oauth password, should include the User password - #var.password: "P@$$W0₹D" + var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + var.token_url: "https://login.salesforce.com" + var.user: "abc.xyz@mail.com" + var.password: "P@$$W0₹D" - # URL, should include the instance_url - #var.url: "https://instance_id.my.salesforce.com" + var.url: "https://instance_id.my.salesforce.com" - # Interval, should include the time interval - #var.interval: 1h \ No newline at end of file + var.real_time: true + var.real_time_interval: 5m diff --git a/x-pack/filebeat/module/salesforce/apex-rest/config/apex-rest.yml b/x-pack/filebeat/module/salesforce/apex-rest/config/apex-rest.yml deleted file mode 100644 index 5a5af4ab5d35..000000000000 --- a/x-pack/filebeat/module/salesforce/apex-rest/config/apex-rest.yml +++ /dev/null @@ -1,71 +0,0 @@ -{{ if eq .input "httpjson" }} - -type: httpjson -interval: {{ .interval }} -request.method: GET -auth.oauth2: - enabled: true - {{ if .client_id }} - client.id: {{ .client_id }} - {{ end }} - {{ if .client_secret }} - client.secret: {{ .client_secret }} - {{ end }} - {{ if .token_url }} - token_url: {{ .token_url }} - {{ end }} - {{ if .user }} - user: {{ .user }} - {{ end }} - {{ if .password }} - password: {{ .password }} - {{ end }} -# Query params will be overwritten by request.transforms from start of the input and -# it is to indicate that this url is for Apex type of events as cursor stores the url as source. -# Each filebeat input cursor source needs to be uniquely identified with a name. -request.url: {{ .url }}/services/data/v52.0/query?q=apex+rest -{{ if .proxy_url }} -request.proxy_url: {{ .proxy_url }} -{{ end }} -request.transforms: - - set: - target: url.params.q - value: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND LogDate > [[.cursor.last_published_apex]] AND (EventType = 'ApexCallout' OR EventType = 'ApexExecution' OR EventType = 'ApexRestApi' OR EventType = 'ApexSoap' OR EventType = 'ApexTrigger' OR EventType = 'ExternalCustomApexCallout') ORDER BY LogDate ASC NULLS FIRST" - default: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND (EventType = 'ApexCallout' OR EventType = 'ApexExecution' OR EventType = 'ApexRestApi' OR EventType = 'ApexSoap' OR EventType = 'ApexTrigger' OR EventType = 'ExternalCustomApexCallout') ORDER BY LogDate ASC NULLS FIRST" -response.pagination: - - set: - target: url.value - value: '[[if (ne .last_response.body.done true)]]{{ .url }}[[.last_response.body.nextRecordsUrl]][[end]]' - fail_on_template_error: true -chain: - - step: - request.url: {{ .url }}/services/data/v52.0/sobjects/EventLogFile/$.records[:].Id/LogFile - request.method: GET - replace: $.records[:].Id -cursor: - last_published_apex: - value: '[[(formatDate ((parseDate .last_event.TIMESTAMP_DERIVED "RFC3339").Add (parseDuration "-1h")))]]' - -{{ else if eq .input "file" }} - -type: log -paths: -{{ range $i, $path := .paths }} - - {{$path}} -{{ end }} -exclude_files: [".gz$"] - -{{ end }} - -tags: {{.tags | tojson}} -publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} - -processors: - - decode_json_fields: - fields: [message] - target: "json" - - add_fields: - target: '' - fields: - ecs.version: 1.12.0 - - add_locale: ~ diff --git a/x-pack/filebeat/module/salesforce/apex-rest/manifest.yml b/x-pack/filebeat/module/salesforce/apex-rest/manifest.yml deleted file mode 100644 index 54b0495bb70b..000000000000 --- a/x-pack/filebeat/module/salesforce/apex-rest/manifest.yml +++ /dev/null @@ -1,20 +0,0 @@ -module_version: 1.0 - -var: - - name: input - default: httpjson - - name: interval - default: 1h - - name: tags - default: [salesforce-apex-rest,forwarded] - - name: proxy_url - - name: client_id - - name: client_secret - - name: token_url - - name: user - - name: password - - name: url - -ingest_pipeline: - - ingest/pipeline.yml -input: config/apex-rest.yml diff --git a/x-pack/filebeat/module/salesforce/apex-rest/_meta/fields.yml b/x-pack/filebeat/module/salesforce/apex/_meta/fields.yml similarity index 100% rename from x-pack/filebeat/module/salesforce/apex-rest/_meta/fields.yml rename to x-pack/filebeat/module/salesforce/apex/_meta/fields.yml diff --git a/x-pack/filebeat/module/salesforce/apex/config/apex.yml b/x-pack/filebeat/module/salesforce/apex/config/apex.yml new file mode 100644 index 000000000000..4e8854e5e261 --- /dev/null +++ b/x-pack/filebeat/module/salesforce/apex/config/apex.yml @@ -0,0 +1,79 @@ +{{ if eq .input "salesforce" }} + +type: salesforce +enabled: true +{{ if .api_version }} +version: {{ .api_version }} +{{ end }} +auth.oauth2: +{{ if .jwt_enabled }} + jwt_bearer_flow: + enabled: true + {{ if .client_id }} + client.id: {{ .client_id }} + {{ end }} + {{ if .client_username }} + client.username: {{ .client_username }} + {{ end }} + {{ if .client_key_path }} + client.key_path: {{ .client_key_path }} + {{ end }} + {{ if .jwt_url }} + url: {{ .jwt_url }} + {{ end }} +{{ else }} + user_password_flow: + enabled: true + {{ if .client_id }} + client.id: {{ .client_id }} + {{ end }} + {{ if .client_secret }} + client.secret: {{ .client_secret }} + {{ end }} + {{ if .token_url }} + token_url: {{ .token_url }} + {{ end }} + {{ if .username }} + username: {{ .username }} + {{ end }} + {{ if .password }} + password: {{ .password }} + {{ end }} +{{ end }} +# Query params will be overwritten by request.transforms from start of the input and +# it is to indicate that this url is for Apex type of events as cursor stores the url as source. +# Each filebeat input cursor source needs to be uniquely identified with a name. +url: {{ .url }} +event_monitoring_method: + event_log_file: + enabled: true + interval: {{ .elf_interval }} + query: + default: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND (EventType = 'ApexCallout' OR EventType = 'ApexExecution' OR EventType = 'ApexRestApi' OR EventType = 'ApexSoap' OR EventType = 'ApexTrigger' OR EventType = 'ExternalCustomApexCallout') ORDER BY LogDate ASC NULLS FIRST" + value: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND CreatedDate > [[ .cursor.event_log_file.last_event_time ]] AND (EventType = 'ApexCallout' OR EventType = 'ApexExecution' OR EventType = 'ApexRestApi' OR EventType = 'ApexSoap' OR EventType = 'ApexTrigger' OR EventType = 'ExternalCustomApexCallout') ORDER BY LogDate ASC NULLS FIRST" + cursor: + field: "CreatedDate" + +{{ else if eq .input "file" }} + +type: log +paths: +{{ range $i, $path := .paths }} + - {{$path}} +{{ end }} +exclude_files: [".gz$"] + +{{ end }} + +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} + +processors: + - decode_json_fields: + fields: [message] + target: "json" + - add_fields: + target: '' + fields: + ecs.version: 1.12.0 + - add_locale: ~ diff --git a/x-pack/filebeat/module/salesforce/apex-rest/ingest/pipeline.yml b/x-pack/filebeat/module/salesforce/apex/ingest/pipeline.yml similarity index 100% rename from x-pack/filebeat/module/salesforce/apex-rest/ingest/pipeline.yml rename to x-pack/filebeat/module/salesforce/apex/ingest/pipeline.yml diff --git a/x-pack/filebeat/module/salesforce/apex/manifest.yml b/x-pack/filebeat/module/salesforce/apex/manifest.yml new file mode 100644 index 000000000000..97211b436d76 --- /dev/null +++ b/x-pack/filebeat/module/salesforce/apex/manifest.yml @@ -0,0 +1,24 @@ +module_version: 1.0 + +var: + - name: input + default: salesforce + - name: tags + default: [salesforce-apex,forwarded] + - name: api_version + - name: jwt_enabled + - name: client_id + - name: client_username + - name: client_key_path + - name: jwt_url + - name: client_secret + - name: token_url + - name: user + - name: password + - name: url + - name: elf_interval + default: 1h + +ingest_pipeline: + - ingest/pipeline.yml +input: config/apex.yml diff --git a/x-pack/filebeat/module/salesforce/apex-rest/test/salesforce_apex_callout.ndjson.log b/x-pack/filebeat/module/salesforce/apex/test/salesforce_apex_callout.ndjson.log similarity index 100% rename from x-pack/filebeat/module/salesforce/apex-rest/test/salesforce_apex_callout.ndjson.log rename to x-pack/filebeat/module/salesforce/apex/test/salesforce_apex_callout.ndjson.log diff --git a/x-pack/filebeat/module/salesforce/apex-rest/test/salesforce_apex_callout.ndjson.log-expected.json b/x-pack/filebeat/module/salesforce/apex/test/salesforce_apex_callout.ndjson.log-expected.json similarity index 100% rename from x-pack/filebeat/module/salesforce/apex-rest/test/salesforce_apex_callout.ndjson.log-expected.json rename to x-pack/filebeat/module/salesforce/apex/test/salesforce_apex_callout.ndjson.log-expected.json diff --git a/x-pack/filebeat/module/salesforce/apex-rest/test/salesforce_apex_execution.ndjson.log b/x-pack/filebeat/module/salesforce/apex/test/salesforce_apex_execution.ndjson.log similarity index 100% rename from x-pack/filebeat/module/salesforce/apex-rest/test/salesforce_apex_execution.ndjson.log rename to x-pack/filebeat/module/salesforce/apex/test/salesforce_apex_execution.ndjson.log diff --git a/x-pack/filebeat/module/salesforce/apex-rest/test/salesforce_apex_execution.ndjson.log-expected.json b/x-pack/filebeat/module/salesforce/apex/test/salesforce_apex_execution.ndjson.log-expected.json similarity index 100% rename from x-pack/filebeat/module/salesforce/apex-rest/test/salesforce_apex_execution.ndjson.log-expected.json rename to x-pack/filebeat/module/salesforce/apex/test/salesforce_apex_execution.ndjson.log-expected.json diff --git a/x-pack/filebeat/module/salesforce/apex-rest/test/salesforce_apex_external_callout.ndjson.log b/x-pack/filebeat/module/salesforce/apex/test/salesforce_apex_external_callout.ndjson.log similarity index 100% rename from x-pack/filebeat/module/salesforce/apex-rest/test/salesforce_apex_external_callout.ndjson.log rename to x-pack/filebeat/module/salesforce/apex/test/salesforce_apex_external_callout.ndjson.log diff --git a/x-pack/filebeat/module/salesforce/apex-rest/test/salesforce_apex_external_callout.ndjson.log-expected.json b/x-pack/filebeat/module/salesforce/apex/test/salesforce_apex_external_callout.ndjson.log-expected.json similarity index 100% rename from x-pack/filebeat/module/salesforce/apex-rest/test/salesforce_apex_external_callout.ndjson.log-expected.json rename to x-pack/filebeat/module/salesforce/apex/test/salesforce_apex_external_callout.ndjson.log-expected.json diff --git a/x-pack/filebeat/module/salesforce/apex-rest/test/salesforce_apex_rest.ndjson.log b/x-pack/filebeat/module/salesforce/apex/test/salesforce_apex_rest.ndjson.log similarity index 100% rename from x-pack/filebeat/module/salesforce/apex-rest/test/salesforce_apex_rest.ndjson.log rename to x-pack/filebeat/module/salesforce/apex/test/salesforce_apex_rest.ndjson.log diff --git a/x-pack/filebeat/module/salesforce/apex-rest/test/salesforce_apex_rest.ndjson.log-expected.json b/x-pack/filebeat/module/salesforce/apex/test/salesforce_apex_rest.ndjson.log-expected.json similarity index 100% rename from x-pack/filebeat/module/salesforce/apex-rest/test/salesforce_apex_rest.ndjson.log-expected.json rename to x-pack/filebeat/module/salesforce/apex/test/salesforce_apex_rest.ndjson.log-expected.json diff --git a/x-pack/filebeat/module/salesforce/apex-rest/test/salesforce_apex_soap.ndjson.log b/x-pack/filebeat/module/salesforce/apex/test/salesforce_apex_soap.ndjson.log similarity index 100% rename from x-pack/filebeat/module/salesforce/apex-rest/test/salesforce_apex_soap.ndjson.log rename to x-pack/filebeat/module/salesforce/apex/test/salesforce_apex_soap.ndjson.log diff --git a/x-pack/filebeat/module/salesforce/apex-rest/test/salesforce_apex_soap.ndjson.log-expected.json b/x-pack/filebeat/module/salesforce/apex/test/salesforce_apex_soap.ndjson.log-expected.json similarity index 100% rename from x-pack/filebeat/module/salesforce/apex-rest/test/salesforce_apex_soap.ndjson.log-expected.json rename to x-pack/filebeat/module/salesforce/apex/test/salesforce_apex_soap.ndjson.log-expected.json diff --git a/x-pack/filebeat/module/salesforce/apex-rest/test/salesforce_apex_trigger.ndjson.log b/x-pack/filebeat/module/salesforce/apex/test/salesforce_apex_trigger.ndjson.log similarity index 100% rename from x-pack/filebeat/module/salesforce/apex-rest/test/salesforce_apex_trigger.ndjson.log rename to x-pack/filebeat/module/salesforce/apex/test/salesforce_apex_trigger.ndjson.log diff --git a/x-pack/filebeat/module/salesforce/apex-rest/test/salesforce_apex_trigger.ndjson.log-expected.json b/x-pack/filebeat/module/salesforce/apex/test/salesforce_apex_trigger.ndjson.log-expected.json similarity index 100% rename from x-pack/filebeat/module/salesforce/apex-rest/test/salesforce_apex_trigger.ndjson.log-expected.json rename to x-pack/filebeat/module/salesforce/apex/test/salesforce_apex_trigger.ndjson.log-expected.json diff --git a/x-pack/filebeat/module/salesforce/login-rest/config/login-rest.yml b/x-pack/filebeat/module/salesforce/login-rest/config/login-rest.yml deleted file mode 100644 index 380c746d7394..000000000000 --- a/x-pack/filebeat/module/salesforce/login-rest/config/login-rest.yml +++ /dev/null @@ -1,71 +0,0 @@ -{{ if eq .input "httpjson" }} - -type: httpjson -interval: {{ .interval }} -request.method: GET -auth.oauth2: - enabled: true - {{ if .client_id }} - client.id: {{ .client_id }} - {{ end }} - {{ if .client_secret }} - client.secret: {{ .client_secret }} - {{ end }} - {{ if .token_url }} - token_url: {{ .token_url }} - {{ end }} - {{ if .user }} - user: {{ .user }} - {{ end }} - {{ if .password }} - password: {{ .password }} - {{ end }} -# Query params will be overwritten by request.transforms from start of the input and -# it is to indicate that this url is for Login type of events as cursor stores the url as source. -# Each filebeat input cursor source needs to be uniquely identified with a name. -request.url: {{ .url }}/services/data/v52.0/query?q=login+rest -{{ if .proxy_url }} -request.proxy_url: {{ .proxy_url }} -{{ end }} -request.transforms: - - set: - target: url.params.q - value: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Login' AND LogDate > [[.cursor.last_published_login]] ORDER BY LogDate ASC NULLS FIRST" - default: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Login' ORDER BY LogDate ASC NULLS FIRST" -response.pagination: - - set: - target: url.value - value: '[[if (ne .last_response.body.done true)]]{{ .url }}[[.last_response.body.nextRecordsUrl]][[end]]' - fail_on_template_error: true -chain: - - step: - request.url: {{ .url }}/services/data/v52.0/sobjects/EventLogFile/$.records[:].Id/LogFile - request.method: GET - replace: $.records[:].Id -cursor: - last_published_login: - value: '[[(formatDate ((parseDate .last_event.TIMESTAMP_DERIVED "RFC3339").Add (parseDuration "-1h")))]]' - -{{ else if eq .input "file" }} - -type: log -paths: -{{ range $i, $path := .paths }} - - {{$path}} -{{ end }} -exclude_files: [".gz$"] - -{{ end }} - -tags: {{.tags | tojson}} -publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} - -processors: - - decode_json_fields: - fields: [message] - target: "json" - - add_fields: - target: '' - fields: - ecs.version: 1.12.0 - - add_locale: ~ diff --git a/x-pack/filebeat/module/salesforce/login-rest/manifest.yml b/x-pack/filebeat/module/salesforce/login-rest/manifest.yml deleted file mode 100644 index 15450bab7f28..000000000000 --- a/x-pack/filebeat/module/salesforce/login-rest/manifest.yml +++ /dev/null @@ -1,20 +0,0 @@ -module_version: 1.0 - -var: - - name: input - default: httpjson - - name: interval - default: 1h - - name: tags - default: [salesforce-login-rest, forwarded] - - name: proxy_url - - name: client_id - - name: client_secret - - name: token_url - - name: user - - name: password - - name: url - -ingest_pipeline: - - ingest/pipeline.yml -input: config/login-rest.yml diff --git a/x-pack/filebeat/module/salesforce/login-stream/_meta/fields.yml b/x-pack/filebeat/module/salesforce/login-stream/_meta/fields.yml deleted file mode 100644 index 9d807a7bf778..000000000000 --- a/x-pack/filebeat/module/salesforce/login-stream/_meta/fields.yml +++ /dev/null @@ -1,58 +0,0 @@ -- name: salesforce.login - type: group - release: beta - description: > - Fileset for ingesting Salesforce Login (Streaming) logs. - fields: - - name: application - type: keyword - description: > - The application used to access the org. Possible values include: AppExchange, Browser, Salesforce for iOS, Salesforce Developers API Explorer, N/A - - name: auth_method_reference - type: keyword - description: > - The authentication method used by a third-party identification provider for an OpenID Connect single sign-on protocol. - - name: auth_service_id - type: keyword - description: > - The 18-character ID for an authentication service for a login event. - - name: client_version - type: keyword - description: > - The version number of the login client. If no version number is available, “Unknown” is returned. - - name: created_by_id - type: keyword - description: > - Unavailable - - name: evaluation_time - type: keyword - description: > - The amount of time it took to evaluate the transaction security policy, in milliseconds. - - name: login_geo_id - type: keyword - description: > - The Salesforce ID of the LoginGeo object associated with the login user’s IP address. - - name: login_history_id - type: keyword - description: > - Tracks a user session so you can correlate user activity with a particular login instance. This field is also available on the LoginHistory, AuthSession, and LoginHistory objects, making it easier to trace events back to a user’s original authentication. - - name: login_type - type: keyword - description: > - The type of login used to access the session. - - name: policy_id - type: keyword - description: > - The ID of the transaction security policy associated with this event. - - name: policy_outcome - type: keyword - description: > - The result of the transaction policy. - - name: related_event_identifier - type: keyword - description: > - This field is populated only when the activity that this event monitors requires extra authentication, such as multi-factor authentication. In this case, Salesforce generates more events and sets the RelatedEventIdentifier field of the new events to the value of the EventIdentifier field of the original event. Use this field with the EventIdentifier field to correlate all the related events. If no extra authentication is required, this field is blank. - - name: session_level - type: keyword - description: > - Session-level security controls user access to features that support it, such as connected apps and reporting. Possible values are: HIGH_ASSURANCE, LOW, STANDARD \ No newline at end of file diff --git a/x-pack/filebeat/module/salesforce/login-stream/config/login-stream.yml b/x-pack/filebeat/module/salesforce/login-stream/config/login-stream.yml deleted file mode 100644 index 3db5b0dd49ed..000000000000 --- a/x-pack/filebeat/module/salesforce/login-stream/config/login-stream.yml +++ /dev/null @@ -1,44 +0,0 @@ -{{ if eq .input "cometd" }} - -type: cometd -channel_name: /event/LoginEventStream -auth.oauth2: - {{ if .client_id }} - client.id: {{ .client_id }} - {{ end }} - {{ if .client_secret }} - client.secret: {{ .client_secret }} - {{ end }} - {{ if .token_url }} - token_url: {{ .token_url }} - {{ end }} - {{ if .user }} - user: {{ .user }} - {{ end }} - {{ if .password }} - password: {{ .password }} - {{ end }} - -{{ else if eq .input "file" }} - -type: log -paths: -{{ range $i, $path := .paths }} - - {{$path}} -{{ end }} -exclude_files: [".gz$"] - -{{ end }} - -tags: {{.tags | tojson}} -publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} - -processors: - - decode_json_fields: - fields: [message] - target: "json" - - add_fields: - target: '' - fields: - ecs.version: 1.12.0 - - add_locale: ~ diff --git a/x-pack/filebeat/module/salesforce/login-stream/ingest/pipeline.yml b/x-pack/filebeat/module/salesforce/login-stream/ingest/pipeline.yml deleted file mode 100644 index a24e65d1a212..000000000000 --- a/x-pack/filebeat/module/salesforce/login-stream/ingest/pipeline.yml +++ /dev/null @@ -1,278 +0,0 @@ ---- -description: Pipeline for parsing Salesforce Login (Streaming) logs -processors: -- set: - field: event.ingested - value: '{{_ingest.timestamp}}' -- remove: - field: - - message - ignore_missing: true - -- set: - field: salesforce.access_mode - value: stream - ignore_failure: true - -- date: - field: json.EventDate - target_field: "@timestamp" - formats: - - ISO8601 - ignore_failure: true -- rename: - field: json.schema - target_field: salesforce.login.schema - ignore_missing: true -- rename: - field: json.ApiType - target_field: salesforce.login.api_type - ignore_missing: true -- rename: - field: json.AuthMethodReference - target_field: salesforce.login.auth_method_reference - ignore_missing: true -- rename: - field: json.LoginType - target_field: salesforce.login.login_type - ignore_missing: true -- rename: - field: json.PolicyOutcome - target_field: salesforce.login.policy_outcome - ignore_missing: true -- rename: - field: json.AuthServiceId - target_field: salesforce.login.auth_service_id - ignore_missing: true -- rename: - field: json.EvaluationTime - target_field: salesforce.login.evaluation_time - ignore_missing: true -- rename: - field: json.ClientVersion - target_field: salesforce.login.client_version - ignore_missing: true -- rename: - field: json.LoginGeoId - target_field: salesforce.login.login_geo_id - ignore_missing: true -- rename: - field: json.LoginHistoryId - target_field: salesforce.login.login_history_id - ignore_missing: true -- rename: - field: json.CreatedById - target_field: salesforce.login.created_by_id - ignore_missing: true -- rename: - field: json.ApiVersion - target_field: salesforce.login.api_version - ignore_missing: true -- rename: - field: json.RelatedEventIdentifier - target_field: salesforce.login.related_event_identifier - ignore_missing: true -- rename: - field: json.LoginKey - target_field: salesforce.login.login_key - ignore_missing: true -- rename: - field: json.Application - target_field: salesforce.login.application - ignore_missing: true -- rename: - field: json.PolicyId - target_field: salesforce.login.policy_id - ignore_missing: true -- rename: - field: json.SessionLevel - target_field: salesforce.login.session_level - ignore_missing: true -- rename: - field: json.replayId - target_field: salesforce.login.replay_id - ignore_missing: true - -####################### -## ECS Event Mapping ## -####################### - -- set: - field: event.outcome - value: success - if: 'ctx?.json?.Status == "Success" && ctx?.json?.Status != null' - ignore_failure: true -- set: - field: event.outcome - value: failure - if: 'ctx?.json?.Status != "Success" && ctx?.json?.Status != null' - ignore_failure: true -- date: - field: json.CreatedDate - target_field: event.created - formats: - - ISO8601 - ignore_failure: true -- rename: - field: json.LoginUrl - target_field: event.url - ignore_missing: true -- set: - field: event.type - value: "info" -- set: - field: event.kind - value: "event" -- set: - field: event.action - value: "login-attempt" -- set: - field: event.category - value: "authentication" -- set: - field: event.dataset - value: "salesforce.login" -- set: - field: event.module - value: "salesforce" - -###################### -## ECS User Mapping ## -###################### - -# As per the following article, the username must be in the format of an email address. -# Reference: https://help.salesforce.com/s/articleView?language=en_US&type=5&id=sf.basics_intro_usernames_passwords.htm -- rename: - field: json.Username - target_field: user.email - ignore_missing: true -- rename: - field: json.UserId - target_field: user.id - ignore_missing: true -- rename: - field: json.UserType - target_field: user.roles - ignore_missing: true - -######################## -## ECS Source Mapping ## -######################## - -- rename: - field: json.SourceIp - target_field: source.ip - ignore_missing: true - -############################ -## ECS Related.ip Mapping ## -############################ - -- append: - field: related.ip - value: "{{{source.ip}}}" - if: ctx?.source?.ip != null - allow_duplicates: false - ignore_failure: true - -############################ -## ECS Source.Geo Mapping ## -############################ - -- rename: - field: json.LoginLatitude - target_field: source.geo.location.lat - ignore_missing: true -- rename: - field: json.LoginLongitude - target_field: source.geo.location.lon - ignore_missing: true -- rename: - field: json.CountryIso - target_field: source.geo.country_iso_code - ignore_missing: true -- rename: - field: json.PostalCode - target_field: source.geo.postal_code - ignore_missing: true -- rename: - field: json.City - target_field: source.geo.city_name - ignore_missing: true -- rename: - field: json.Subdivision - target_field: source.geo.region_name - ignore_missing: true -- rename: - field: json.Country - target_field: source.geo.country_name - ignore_missing: true - -############################ -## ECS User Agent Mapping ## -############################ - -- rename: - field: json.Browser - target_field: user_agent.name - ignore_missing: true -- rename: - field: json.Platform - target_field: user_agent.os.platform - ignore_missing: true - -###################### -## ECS HTTP Mapping ## -###################### - -- rename: - field: json.HttpMethod - target_field: http.request.method - ignore_missing: true -- rename: - field: json.AdditionalInfo - target_field: http.request.body.content - ignore_missing: true - -##################### -## ECS TLS Mapping ## -##################### - -- rename: - field: json.CipherSuite - target_field: tls.cipher - ignore_missing: true -- dissect: - pattern: "%{tls.version_protocol} %{tls.version}" - field: "json.TlsProtocol" - ignore_failure: true - -############# -## Cleanup ## -############# - -- script: - description: Drops null/empty values recursively - lang: painless - source: | - boolean dropEmptyFields(Object object) { - if (object == null || object == "") { - return true; - } else if (object instanceof Map) { - ((Map) object).values().removeIf(value -> dropEmptyFields(value)); - return (((Map) object).size() == 0); - } else if (object instanceof List) { - ((List) object).removeIf(value -> dropEmptyFields(value)); - return (((List) object).length == 0); - } - return false; - } - dropEmptyFields(ctx); -- remove: - field: - - json - ignore_missing: true -on_failure: -- set: - field: error.message - value: '{{_ingest.on_failure_message}}' diff --git a/x-pack/filebeat/module/salesforce/login-stream/manifest.yml b/x-pack/filebeat/module/salesforce/login-stream/manifest.yml deleted file mode 100644 index 92411aa750ea..000000000000 --- a/x-pack/filebeat/module/salesforce/login-stream/manifest.yml +++ /dev/null @@ -1,16 +0,0 @@ -module_version: 1.0 - -var: - - name: input - default: cometd - - name: tags - default: [salesforce-login-stream, forwarded] - - name: client_id - - name: client_secret - - name: token_url - - name: user - - name: password - -ingest_pipeline: - - ingest/pipeline.yml -input: config/login-stream.yml diff --git a/x-pack/filebeat/module/salesforce/login-stream/test/salesforce_login.ndjson.log b/x-pack/filebeat/module/salesforce/login-stream/test/salesforce_login.ndjson.log deleted file mode 100644 index 53d856e9d360..000000000000 --- a/x-pack/filebeat/module/salesforce/login-stream/test/salesforce_login.ndjson.log +++ /dev/null @@ -1 +0,0 @@ -{"EventDate":"2021-10-19T11:47:22Z","AuthServiceId":null,"CountryIso":"IN","Platform":"Unknown","EvaluationTime":0,"CipherSuite":"ECDHE-RSA-AES256-GCM-SHA384","PostalCode":"395007","ClientVersion":"N/A","LoginGeoId":"04F5j00000FadrI","LoginUrl":"login.salesforce.com","LoginHistoryId":"0Ya5j00000GLxCdCAL","CreatedById":"0055j000000q9s7AAA","SessionKey":null,"ApiType":"N/A","AuthMethodReference":null,"LoginType":"Remote Access 2.0","PolicyOutcome":null,"Status":"Success","AdditionalInfo":"{}","ApiVersion":"N/A","EventIdentifier":"06af6d92-1167-467d-a826-ee8583f7134d","RelatedEventIdentifier":null,"LoginLatitude":21.1888,"City":"Surat","Subdivision":"Gujarat","SourceIp":"123.201.231.106","Username":"kush.rana@elastic.co","UserId":"0055j000000utlPAAQ","CreatedDate":"2021-10-19T11:47:30Z","Country":"India","LoginLongitude":72.8293,"TlsProtocol":"TLS 1.2","LoginKey":"o3vhFaSRBb0OzpCl","Application":"elastic integration","UserType":"Standard","PolicyId":null,"HttpMethod":"POST","SessionLevel":"STANDARD","Browser":"Unknown"} diff --git a/x-pack/filebeat/module/salesforce/login-stream/test/salesforce_login.ndjson.log-expected.json b/x-pack/filebeat/module/salesforce/login-stream/test/salesforce_login.ndjson.log-expected.json deleted file mode 100644 index 564108c2357b..000000000000 --- a/x-pack/filebeat/module/salesforce/login-stream/test/salesforce_login.ndjson.log-expected.json +++ /dev/null @@ -1,55 +0,0 @@ -[ - { - "@timestamp": "2021-10-19T11:47:22.000Z", - "event.action": "login-attempt", - "event.category": "authentication", - "event.dataset": "salesforce.login", - "event.kind": "event", - "event.module": "salesforce", - "event.outcome": "success", - "event.timezone": "-02:00", - "event.type": "info", - "event.url": "login.salesforce.com", - "fileset.name": "login-stream", - "http.request.body.content": "{}", - "http.request.method": "POST", - "input.type": "log", - "log.offset": 0, - "related.ip": [ - "123.201.231.106" - ], - "salesforce.access_mode": "stream", - "salesforce.login.api_type": "N/A", - "salesforce.login.api_version": "N/A", - "salesforce.login.application": "elastic integration", - "salesforce.login.client_version": "N/A", - "salesforce.login.created_by_id": "0055j000000q9s7AAA", - "salesforce.login.evaluation_time": 0, - "salesforce.login.login_geo_id": "04F5j00000FadrI", - "salesforce.login.login_history_id": "0Ya5j00000GLxCdCAL", - "salesforce.login.login_key": "o3vhFaSRBb0OzpCl", - "salesforce.login.login_type": "Remote Access 2.0", - "salesforce.login.session_level": "STANDARD", - "service.type": "salesforce", - "source.geo.city_name": "Surat", - "source.geo.country_iso_code": "IN", - "source.geo.country_name": "India", - "source.geo.location.lat": 21.1888, - "source.geo.location.lon": 72.8293, - "source.geo.postal_code": "395007", - "source.geo.region_name": "Gujarat", - "source.ip": "123.201.231.106", - "tags": [ - "forwarded", - "salesforce-login-stream" - ], - "tls.cipher": "ECDHE-RSA-AES256-GCM-SHA384", - "tls.version": "1.2", - "tls.version_protocol": "TLS", - "user.email": "kush.rana@elastic.co", - "user.id": "0055j000000utlPAAQ", - "user.roles": "Standard", - "user_agent.name": "Unknown", - "user_agent.os.platform": "Unknown" - } -] \ No newline at end of file diff --git a/x-pack/filebeat/module/salesforce/login-rest/_meta/fields.yml b/x-pack/filebeat/module/salesforce/login/_meta/fields.yml similarity index 100% rename from x-pack/filebeat/module/salesforce/login-rest/_meta/fields.yml rename to x-pack/filebeat/module/salesforce/login/_meta/fields.yml diff --git a/x-pack/filebeat/module/salesforce/login/config/login.yml b/x-pack/filebeat/module/salesforce/login/config/login.yml new file mode 100644 index 000000000000..b7b9a27b439b --- /dev/null +++ b/x-pack/filebeat/module/salesforce/login/config/login.yml @@ -0,0 +1,95 @@ +{{ if eq .input "salesforce" }} + +type: salesforce +enabled: true +{{ if .api_version }} +version: {{ .api_version }} +{{ end }} +auth.oauth2: +{{ if .jwt_enabled }} + jwt_bearer_flow: + enabled: true + {{ if .client_id }} + client.id: {{ .client_id }} + {{ end }} + {{ if .client_username }} + client.username: {{ .client_username }} + {{ end }} + {{ if .client_key_path }} + client.key_path: {{ .client_key_path }} + {{ end }} + {{ if .jwt_url }} + url: {{ .jwt_url }} + {{ end }} +{{ else }} + user_password_flow: + enabled: true + {{ if .client_id }} + client.id: {{ .client_id }} + {{ end }} + {{ if .client_secret }} + client.secret: {{ .client_secret }} + {{ end }} + {{ if .token_url }} + token_url: {{ .token_url }} + {{ end }} + {{ if .username }} + username: {{ .username }} + {{ end }} + {{ if .password }} + password: {{ .password }} + {{ end }} +{{ end }} +# Query params will be overwritten by request.transforms from start of the input and +# it is to indicate that this url is for Apex type of events as cursor stores the url as source. +# Each filebeat input cursor source needs to be uniquely identified with a name. +url: {{ .url }} +event_monitoring_method: + event_log_file: + {{ if .event_log_file }} + enabled: true + {{ else }} + enabled: false + {{ end }} + interval: {{ .elf_interval }} + query: + default: "SELECT CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Login' ORDER BY LogDate ASC NULLS FIRST" + value: "SELECT CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Login' AND CreatedDate > [[ .cursor.event_log_file.last_event_time ]] ORDER BY LogDate ASC NULLS FIRST" + cursor: + field: "CreatedDate" + object: + {{ if .object }} + enabled: true + {{ else }} + enabled: false + {{ end }} + interval: {{ .real_time_interval }} + query: + default: "SELECT FIELDS(STANDARD) FROM LoginEvent" + value: "SELECT FIELDS(STANDARD) FROM LoginEvent WHERE EventDate > [[ .cursor.object.first_event_time ]]" + cursor: + field: "EventDate" + +{{ else if eq .input "file" }} + +type: log +paths: +{{ range $i, $path := .paths }} + - {{$path}} +{{ end }} +exclude_files: [".gz$"] + +{{ end }} + +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} + +processors: + - decode_json_fields: + fields: [message] + target: "json" + - add_fields: + target: '' + fields: + ecs.version: 1.12.0 + - add_locale: ~ diff --git a/x-pack/filebeat/module/salesforce/login-rest/ingest/pipeline.yml b/x-pack/filebeat/module/salesforce/login/ingest/pipeline.yml similarity index 100% rename from x-pack/filebeat/module/salesforce/login-rest/ingest/pipeline.yml rename to x-pack/filebeat/module/salesforce/login/ingest/pipeline.yml diff --git a/x-pack/filebeat/module/salesforce/login/manifest.yml b/x-pack/filebeat/module/salesforce/login/manifest.yml new file mode 100644 index 000000000000..118a3bdd8867 --- /dev/null +++ b/x-pack/filebeat/module/salesforce/login/manifest.yml @@ -0,0 +1,28 @@ +module_version: 1.0 + +var: + - name: input + default: salesforce + - name: tags + default: [salesforce-login, forwarded] + - name: api_version + - name: jwt_enabled + - name: client_id + - name: client_username + - name: client_key_path + - name: jwt_url + - name: client_secret + - name: token_url + - name: user + - name: password + - name: url + - name: event_log_file + - name: elf_interval + default: 1h + - name: real_time + - name: real_time_interval + default: 5m + +ingest_pipeline: + - ingest/pipeline.yml +input: config/login.yml diff --git a/x-pack/filebeat/module/salesforce/login-rest/test/salesforce_login.ndjson.log b/x-pack/filebeat/module/salesforce/login/test/salesforce_login.ndjson.log similarity index 100% rename from x-pack/filebeat/module/salesforce/login-rest/test/salesforce_login.ndjson.log rename to x-pack/filebeat/module/salesforce/login/test/salesforce_login.ndjson.log diff --git a/x-pack/filebeat/module/salesforce/login-rest/test/salesforce_login.ndjson.log-expected.json b/x-pack/filebeat/module/salesforce/login/test/salesforce_login.ndjson.log-expected.json similarity index 100% rename from x-pack/filebeat/module/salesforce/login-rest/test/salesforce_login.ndjson.log-expected.json rename to x-pack/filebeat/module/salesforce/login/test/salesforce_login.ndjson.log-expected.json diff --git a/x-pack/filebeat/module/salesforce/logout-rest/config/logout-rest.yml b/x-pack/filebeat/module/salesforce/logout-rest/config/logout-rest.yml deleted file mode 100644 index 29e8d8b1a79e..000000000000 --- a/x-pack/filebeat/module/salesforce/logout-rest/config/logout-rest.yml +++ /dev/null @@ -1,70 +0,0 @@ -{{ if eq .input "httpjson" }} - -type: httpjson -interval: {{ .interval }} -request.method: GET -auth.oauth2: - {{ if .client_id }} - client.id: {{ .client_id }} - {{ end }} - {{ if .client_secret }} - client.secret: {{ .client_secret }} - {{ end }} - {{ if .token_url }} - token_url: {{ .token_url }} - {{ end }} - {{ if .user }} - user: {{ .user }} - {{ end }} - {{ if .password }} - password: {{ .password }} - {{ end }} -# Query params will be overwritten by request.transforms from start of the input and -# it is to indicate that this url is for Logout type of events as cursor stores the url as source. -# Each filebeat input cursor source needs to be uniquely identified with a name. -request.url: {{ .url }}/services/data/v52.0/query?q=logout+rest -{{ if .proxy_url }} -request.proxy_url: {{ .proxy_url }} -{{ end }} -request.transforms: - - set: - target: url.params.q - value: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Logout' AND LogDate > [[.cursor.last_published_logout]] ORDER BY LogDate ASC NULLS FIRST" - default: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Logout' ORDER BY LogDate ASC NULLS FIRST" -response.pagination: - - set: - target: url.value - value: '[[if (ne .last_response.body.done true)]]{{ .url }}[[.last_response.body.nextRecordsUrl]][[end]]' - fail_on_template_error: true -chain: - - step: - request.url: {{ .url }}/services/data/v52.0/sobjects/EventLogFile/$.records[:].Id/LogFile - request.method: GET - replace: $.records[:].Id -cursor: - last_published_logout: - value: '[[(formatDate ((parseDate .last_event.TIMESTAMP_DERIVED "RFC3339").Add (parseDuration "-1h")))]]' - -{{ else if eq .input "file" }} - -type: log -paths: -{{ range $i, $path := .paths }} - - {{$path}} -{{ end }} -exclude_files: [".gz$"] - -{{ end }} - -tags: {{.tags | tojson}} -publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} - -processors: - - decode_json_fields: - fields: [message] - target: "json" - - add_fields: - target: '' - fields: - ecs.version: 1.12.0 - - add_locale: ~ diff --git a/x-pack/filebeat/module/salesforce/logout-rest/manifest.yml b/x-pack/filebeat/module/salesforce/logout-rest/manifest.yml deleted file mode 100644 index 775f5094a3d2..000000000000 --- a/x-pack/filebeat/module/salesforce/logout-rest/manifest.yml +++ /dev/null @@ -1,19 +0,0 @@ -module_version: 1.0 - -var: - - name: input - default: httpjson - - name: interval - default: 1h - - name: tags - default: [salesforce-logout-rest, forwarded] - - name: proxy_url - - name: client_id - - name: client_secret - - name: token_url - - name: user - - name: password - - name: url - -ingest_pipeline: ingest/pipeline.yml -input: config/logout-rest.yml diff --git a/x-pack/filebeat/module/salesforce/logout-stream/_meta/fields.yml b/x-pack/filebeat/module/salesforce/logout-stream/_meta/fields.yml deleted file mode 100644 index fe1ade9745d2..000000000000 --- a/x-pack/filebeat/module/salesforce/logout-stream/_meta/fields.yml +++ /dev/null @@ -1,22 +0,0 @@ -- name: salesforce.logout - type: group - release: beta - description: > - Fileset for parsing Salesforce Logout (Streaming) logs. - fields: - - name: created_by_id - type: keyword - description: > - Unavailable - - name: related_event_identifier - type: keyword - description: > - This field is populated only when the activity that this event monitors requires extra authentication, such as multi-factor authentication. In this case, Salesforce generates more events and sets the RelatedEventIdentifier field of the new events to the value of the EventIdentifier field of the original event. Use this field with the EventIdentifier field to correlate all the related events. If no extra authentication is required, this field is blank. - - name: replay_id - type: keyword - description: > - Represents an ID value that is populated by the system and refers to the position of the event in the event stream. Replay ID values aren’t guaranteed to be contiguous for consecutive events. A subscriber can store a replay ID value and use it on resubscription to retrieve missed events that are within the retention window. - - name: schema - type: keyword - description: > - Unavailable \ No newline at end of file diff --git a/x-pack/filebeat/module/salesforce/logout-stream/config/logout-stream.yml b/x-pack/filebeat/module/salesforce/logout-stream/config/logout-stream.yml deleted file mode 100644 index d717f9bf4cb8..000000000000 --- a/x-pack/filebeat/module/salesforce/logout-stream/config/logout-stream.yml +++ /dev/null @@ -1,44 +0,0 @@ -{{ if eq .input "cometd" }} - -type: cometd -channel_name: /event/LogoutEventStream -auth.oauth2: - {{ if .client_id }} - client.id: {{ .client_id }} - {{ end }} - {{ if .client_secret }} - client.secret: {{ .client_secret }} - {{ end }} - {{ if .token_url }} - token_url: {{ .token_url }} - {{ end }} - {{ if .user }} - user: {{ .user }} - {{ end }} - {{ if .password }} - password: {{ .password }} - {{ end }} - -{{ else if eq .input "file" }} - -type: log -paths: -{{ range $i, $path := .paths }} - - {{$path}} -{{ end }} -exclude_files: [".gz$"] - -{{ end }} - -tags: {{.tags | tojson}} -publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} - -processors: - - decode_json_fields: - fields: [message] - target: "json" - - add_fields: - target: '' - fields: - ecs.version: 1.12.0 - - add_locale: ~ diff --git a/x-pack/filebeat/module/salesforce/logout-stream/ingest/pipeline.yml b/x-pack/filebeat/module/salesforce/logout-stream/ingest/pipeline.yml deleted file mode 100644 index 3bc74b68e213..000000000000 --- a/x-pack/filebeat/module/salesforce/logout-stream/ingest/pipeline.yml +++ /dev/null @@ -1,147 +0,0 @@ ---- -description: Pipeline for parsing Salesforce Logout (Streaming) logs -processors: -- set: - field: event.ingested - value: '{{_ingest.timestamp}}' -- remove: - field: - - message - ignore_missing: true - -- set: - field: salesforce.access_mode - value: stream - ignore_failure: true - -- date: - field: json.EventDate - target_field: "@timestamp" - formats: - - ISO8601 - ignore_failure: true - -- rename: - field: json.schema - target_field: salesforce.logout.schema - ignore_missing: true -- rename: - field: json.CreatedById - target_field: salesforce.logout.created_by_id - ignore_missing: true -- rename: - field: json.RelatedEventIdentifier - target_field: salesforce.logout.related_event_identifier - ignore_missing: true -- rename: - field: json.LoginKey - target_field: salesforce.logout.login_key - ignore_missing: true -- rename: - field: json.SessionLevel - target_field: salesforce.logout.session_level - ignore_missing: true -- rename: - field: json.replayId - target_field: salesforce.logout.replay_id - ignore_missing: true - -####################### -## ECS Event Mapping ## -####################### - -- date: - field: json.CreatedDate - target_field: event.created - formats: ["yyyy-MM-ddTHH:mm:ssZ"] - ignore_failure: true -- set: - field: event.type - value: "info" -- set: - field: event.kind - value: "event" -- set: - field: event.action - value: "logout" -- set: - field: event.category - value: "authentication" -- set: - field: event.dataset - value: "salesforce.logout" -- set: - field: event.module - value: "salesforce" - -###################### -## ECS User Mapping ## -###################### - -- rename: - field: json.Username - target_field: user.email - ignore_missing: true -- rename: - field: json.UserId - target_field: user.id - ignore_missing: true - -######################## -## ECS Source Mapping ## -######################## - -- rename: - field: json.SourceIp - target_field: source.ip - ignore_missing: true - -############################ -## ECS Source.Geo Mapping ## -############################ - -- geoip: - field: source.ip - target_field: source.geo - ignore_missing: true - -############################ -## ECS Related.ip Mapping ## -############################ - -- append: - field: related.ip - value: "{{{source.ip}}}" - if: ctx?.source?.ip != null - allow_duplicates: false - ignore_failure: true - -############# -## Cleanup ## -############# - -- script: - description: Drops null/empty values recursively - lang: painless - source: | - boolean dropEmptyFields(Object object) { - if (object == null || object == "") { - return true; - } else if (object instanceof Map) { - ((Map) object).values().removeIf(value -> dropEmptyFields(value)); - return (((Map) object).size() == 0); - } else if (object instanceof List) { - ((List) object).removeIf(value -> dropEmptyFields(value)); - return (((List) object).length == 0); - } - return false; - } - dropEmptyFields(ctx); -- remove: - field: - - json - ignore_missing: true -on_failure: -- set: - field: error.message - value: '{{_ingest.on_failure_message}}' diff --git a/x-pack/filebeat/module/salesforce/logout-stream/manifest.yml b/x-pack/filebeat/module/salesforce/logout-stream/manifest.yml deleted file mode 100644 index b8ffb130f568..000000000000 --- a/x-pack/filebeat/module/salesforce/logout-stream/manifest.yml +++ /dev/null @@ -1,15 +0,0 @@ -module_version: 1.0 - -var: - - name: input - default: cometd - - name: tags - default: [salesforce-logout-stream, forwarded] - - name: client_id - - name: client_secret - - name: token_url - - name: user - - name: password - -ingest_pipeline: ingest/pipeline.yml -input: config/logout-stream.yml diff --git a/x-pack/filebeat/module/salesforce/logout-stream/test/salesforce_logout.ndjson.log b/x-pack/filebeat/module/salesforce/logout-stream/test/salesforce_logout.ndjson.log deleted file mode 100644 index 1234e66a491c..000000000000 --- a/x-pack/filebeat/module/salesforce/logout-stream/test/salesforce_logout.ndjson.log +++ /dev/null @@ -1 +0,0 @@ -{"EventDate":"2021-10-19T11:38:54Z","EventIdentifier":"06ce4a9d-8d6b-4a71-aad8-04d28c9a43df","SourceIp":"123.201.231.106","CreatedById":"0055j000000q9s7AAA","Username":"kush.rana@elastic.co","UserId":"0055j000000utlPAAQ","RelatedEventIdentifier":null,"SessionKey":"6/HAElgoPCwskqBU","CreatedDate":"2021-10-19T11:38:54Z","LoginKey":"CuRVtbMjat6xxbTH","SessionLevel":"STANDARD"} diff --git a/x-pack/filebeat/module/salesforce/logout-stream/test/salesforce_logout.ndjson.log-expected.json b/x-pack/filebeat/module/salesforce/logout-stream/test/salesforce_logout.ndjson.log-expected.json deleted file mode 100644 index b4403515ffc3..000000000000 --- a/x-pack/filebeat/module/salesforce/logout-stream/test/salesforce_logout.ndjson.log-expected.json +++ /dev/null @@ -1,30 +0,0 @@ -[ - { - "@timestamp": "2021-10-19T11:38:54.000Z", - "event.action": "logout", - "event.category": "authentication", - "event.dataset": "salesforce.logout", - "event.kind": "event", - "event.module": "salesforce", - "event.timezone": "-02:00", - "event.type": "info", - "fileset.name": "logout-stream", - "input.type": "log", - "log.offset": 0, - "related.ip": [ - "123.201.231.106" - ], - "salesforce.access_mode": "stream", - "salesforce.logout.created_by_id": "0055j000000q9s7AAA", - "salesforce.logout.login_key": "CuRVtbMjat6xxbTH", - "salesforce.logout.session_level": "STANDARD", - "service.type": "salesforce", - "source.ip": "123.201.231.106", - "tags": [ - "forwarded", - "salesforce-logout-stream" - ], - "user.email": "kush.rana@elastic.co", - "user.id": "0055j000000utlPAAQ" - } -] \ No newline at end of file diff --git a/x-pack/filebeat/module/salesforce/logout-rest/_meta/fields.yml b/x-pack/filebeat/module/salesforce/logout/_meta/fields.yml similarity index 100% rename from x-pack/filebeat/module/salesforce/logout-rest/_meta/fields.yml rename to x-pack/filebeat/module/salesforce/logout/_meta/fields.yml diff --git a/x-pack/filebeat/module/salesforce/logout/config/logout.yml b/x-pack/filebeat/module/salesforce/logout/config/logout.yml new file mode 100644 index 000000000000..64bfee4da87f --- /dev/null +++ b/x-pack/filebeat/module/salesforce/logout/config/logout.yml @@ -0,0 +1,95 @@ +{{ if eq .input "salesforce" }} + +type: salesforce +enabled: true +{{ if .api_version }} +version: {{ .api_version }} +{{ end }} +auth.oauth2: +{{ if .jwt_enabled }} + jwt_bearer_flow: + enabled: true + {{ if .client_id }} + client.id: {{ .client_id }} + {{ end }} + {{ if .client_username }} + client.username: {{ .client_username }} + {{ end }} + {{ if .client_key_path }} + client.key_path: {{ .client_key_path }} + {{ end }} + {{ if .jwt_url }} + url: {{ .jwt_url }} + {{ end }} +{{ else }} + user_password_flow: + enabled: true + {{ if .client_id }} + client.id: {{ .client_id }} + {{ end }} + {{ if .client_secret }} + client.secret: {{ .client_secret }} + {{ end }} + {{ if .token_url }} + token_url: {{ .token_url }} + {{ end }} + {{ if .username }} + username: {{ .username }} + {{ end }} + {{ if .password }} + password: {{ .password }} + {{ end }} +{{ end }} +# Query params will be overwritten by request.transforms from start of the input and +# it is to indicate that this url is for Apex type of events as cursor stores the url as source. +# Each filebeat input cursor source needs to be uniquely identified with a name. +url: {{ .url }} +event_monitoring_method: + event_log_file: + {{ if .event_log_file }} + enabled: true + {{ else }} + enabled: false + {{ end }} + interval: {{ .elf_interval }} + query: + default: "SELECT CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Logout' ORDER BY LogDate ASC NULLS FIRST" + value: "SELECT CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Logout' AND CreatedDate > [[ .cursor.event_log_file.last_event_time ]] ORDER BY LogDate ASC NULLS FIRST" + cursor: + field: "CreatedDate" + object: + {{ if .object }} + enabled: true + {{ else }} + enabled: false + {{ end }} + interval: {{ .real_time_interval }} + query: + default: "SELECT FIELDS(STANDARD) FROM LogoutEvent" + value: "SELECT FIELDS(STANDARD) FROM LogoutEvent WHERE EventDate > [[ .cursor.object.first_event_time ]]" + cursor: + field: "EventDate" + +{{ else if eq .input "file" }} + +type: log +paths: +{{ range $i, $path := .paths }} + - {{$path}} +{{ end }} +exclude_files: [".gz$"] + +{{ end }} + +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} + +processors: + - decode_json_fields: + fields: [message] + target: "json" + - add_fields: + target: '' + fields: + ecs.version: 1.12.0 + - add_locale: ~ diff --git a/x-pack/filebeat/module/salesforce/logout-rest/ingest/pipeline.yml b/x-pack/filebeat/module/salesforce/logout/ingest/pipeline.yml similarity index 100% rename from x-pack/filebeat/module/salesforce/logout-rest/ingest/pipeline.yml rename to x-pack/filebeat/module/salesforce/logout/ingest/pipeline.yml diff --git a/x-pack/filebeat/module/salesforce/logout/manifest.yml b/x-pack/filebeat/module/salesforce/logout/manifest.yml new file mode 100644 index 000000000000..e58449fbafda --- /dev/null +++ b/x-pack/filebeat/module/salesforce/logout/manifest.yml @@ -0,0 +1,27 @@ +module_version: 1.0 + +var: + - name: input + default: salesforce + - name: tags + default: [salesforce-logout, forwarded] + - name: api_version + - name: jwt_enabled + - name: client_id + - name: client_username + - name: client_key_path + - name: jwt_url + - name: client_secret + - name: token_url + - name: user + - name: password + - name: url + - name: event_log_file + - name: elf_interval + default: 1h + - name: real_time + - name: real_time_interval + default: 5m + +ingest_pipeline: ingest/pipeline.yml +input: config/logout.yml diff --git a/x-pack/filebeat/module/salesforce/logout-rest/test/salesforce_logout.ndjson.log b/x-pack/filebeat/module/salesforce/logout/test/salesforce_logout.ndjson.log similarity index 100% rename from x-pack/filebeat/module/salesforce/logout-rest/test/salesforce_logout.ndjson.log rename to x-pack/filebeat/module/salesforce/logout/test/salesforce_logout.ndjson.log diff --git a/x-pack/filebeat/module/salesforce/logout-rest/test/salesforce_logout.ndjson.log-expected.json b/x-pack/filebeat/module/salesforce/logout/test/salesforce_logout.ndjson.log-expected.json similarity index 100% rename from x-pack/filebeat/module/salesforce/logout-rest/test/salesforce_logout.ndjson.log-expected.json rename to x-pack/filebeat/module/salesforce/logout/test/salesforce_logout.ndjson.log-expected.json diff --git a/x-pack/filebeat/module/salesforce/setupaudittrail-rest/config/setupaudittrail-rest.yml b/x-pack/filebeat/module/salesforce/setupaudittrail-rest/config/setupaudittrail-rest.yml deleted file mode 100644 index 34f8f694944a..000000000000 --- a/x-pack/filebeat/module/salesforce/setupaudittrail-rest/config/setupaudittrail-rest.yml +++ /dev/null @@ -1,68 +0,0 @@ -{{ if eq .input "httpjson" }} - -type: httpjson -interval: {{ .interval }} -request.method: GET -auth.oauth2: - enabled: true - {{ if .client_id }} - client.id: {{ .client_id }} - {{ end }} - {{ if .client_secret }} - client.secret: {{ .client_secret }} - {{ end }} - {{ if .token_url }} - token_url: {{ .token_url }} - {{ end }} - {{ if .user }} - user: {{ .user }} - {{ end }} - {{ if .password }} - password: {{ .password }} - {{ end }} -# Query params will be overwritten by request.transforms from start of the input and -# it is to indicate that this url is for SetupAuditTratil type of events as cursor stores the url as source. -# Each filebeat input cursor source needs to be uniquely identified with a name. -request.url: {{ .url }}/services/data/v52.0/query?q=setupaudittrail+rest -{{ if .proxy_url }} -request.proxy_url: {{ .proxy_url }} -{{ end }} -request.transforms: - - set: - target: url.params.q - value: "SELECT Action,CreatedByContext,CreatedById,CreatedByIssuer,CreatedDate,DelegateUser,Display,Id,ResponsibleNamespacePrefix,Section FROM SetupAuditTrail WHERE CreatedDate > [[.cursor.last_published_setupaudittrail]] ORDER BY CreatedDate ASC NULLS FIRST" - default: "SELECT Action,CreatedByContext,CreatedById,CreatedByIssuer,CreatedDate,DelegateUser,Display,Id,ResponsibleNamespacePrefix,Section FROM SetupAuditTrail ORDER BY CreatedDate ASC NULLS FIRST" -response.split: - target: body.records -response.pagination: -- set: - target: url.value - value: '[[if (ne .last_response.body.done true)]]{{ .url }}[[.last_response.body.nextRecordsUrl]][[end]]' - fail_on_template_error: true -cursor: - last_published_setupaudittrail: - value: '[[.last_event.CreatedDate]]' - -{{ else if eq .input "file" }} - -type: log -paths: -{{ range $i, $path := .paths }} - - {{$path}} -{{ end }} -exclude_files: [".gz$"] - -{{ end }} - -tags: {{.tags | tojson}} -publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} - -processors: - - decode_json_fields: - fields: [message] - target: "json" - - add_fields: - target: '' - fields: - ecs.version: 1.12.0 - - add_locale: ~ diff --git a/x-pack/filebeat/module/salesforce/setupaudittrail-rest/manifest.yml b/x-pack/filebeat/module/salesforce/setupaudittrail-rest/manifest.yml deleted file mode 100644 index 889bfe925693..000000000000 --- a/x-pack/filebeat/module/salesforce/setupaudittrail-rest/manifest.yml +++ /dev/null @@ -1,19 +0,0 @@ -module_version: 1.0 - -var: - - name: input - default: httpjson - - name: interval - default: 1h - - name: tags - default: [salesforce-setupaudittrail-rest, forwarded] - - name: proxy_url - - name: client_id - - name: client_secret - - name: token_url - - name: user - - name: password - - name: url - -ingest_pipeline: ingest/pipeline.yml -input: config/setupaudittrail-rest.yml diff --git a/x-pack/filebeat/module/salesforce/setupaudittrail-rest/_meta/fields.yml b/x-pack/filebeat/module/salesforce/setupaudittrail/_meta/fields.yml similarity index 100% rename from x-pack/filebeat/module/salesforce/setupaudittrail-rest/_meta/fields.yml rename to x-pack/filebeat/module/salesforce/setupaudittrail/_meta/fields.yml diff --git a/x-pack/filebeat/module/salesforce/setupaudittrail/config/setupaudittrail.yml b/x-pack/filebeat/module/salesforce/setupaudittrail/config/setupaudittrail.yml new file mode 100644 index 000000000000..e39ed0834437 --- /dev/null +++ b/x-pack/filebeat/module/salesforce/setupaudittrail/config/setupaudittrail.yml @@ -0,0 +1,83 @@ +{{ if eq .input "salesforce" }} + +type: salesforce +enabled: true +{{ if .api_version }} +version: {{ .api_version }} +{{ end }} +auth.oauth2: +{{ if .jwt_enabled }} + jwt_bearer_flow: + enabled: true + {{ if .client_id }} + client.id: {{ .client_id }} + {{ end }} + {{ if .client_username }} + client.username: {{ .client_username }} + {{ end }} + {{ if .client_key_path }} + client.key_path: {{ .client_key_path }} + {{ end }} + {{ if .jwt_url }} + url: {{ .jwt_url }} + {{ end }} +{{ else }} + user_password_flow: + enabled: true + {{ if .client_id }} + client.id: {{ .client_id }} + {{ end }} + {{ if .client_secret }} + client.secret: {{ .client_secret }} + {{ end }} + {{ if .token_url }} + token_url: {{ .token_url }} + {{ end }} + {{ if .username }} + username: {{ .username }} + {{ end }} + {{ if .password }} + password: {{ .password }} + {{ end }} +{{ end }} +# Query params will be overwritten by request.transforms from start of the input and +# it is to indicate that this url is for Apex type of events as cursor stores the url as source. +# Each filebeat input cursor source needs to be uniquely identified with a name. +url: {{ .url }} +event_monitoring_method: + object: + {{ if .object }} + enabled: true + {{ else }} + enabled: false + {{ end }} + interval: {{ .real_time_interval }} + query: + default: "SELECT FIELDS(STANDARD) FROM SetupAuditTrail ORDER BY CreatedDate ASC NULLS FIRST" + value: "SELECT FIELDS(STANDARD) FROM SetupAuditTrail WHERE CreatedDate > [[ .cursor.object.last_event_time ]] ORDER BY CreatedDate ASC NULLS FIRST" + cursor: + field: "CreatedDate" + +{{ else if eq .input "file" }} + +type: log +paths: +{{ range $i, $path := .paths }} + - {{$path}} +{{ end }} +exclude_files: [".gz$"] + +{{ end }} + +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} + +processors: + - decode_json_fields: + fields: [message] + target: "json" + - add_fields: + target: '' + fields: + ecs.version: 1.12.0 + - add_locale: ~ diff --git a/x-pack/filebeat/module/salesforce/setupaudittrail-rest/ingest/pipeline.yml b/x-pack/filebeat/module/salesforce/setupaudittrail/ingest/pipeline.yml similarity index 100% rename from x-pack/filebeat/module/salesforce/setupaudittrail-rest/ingest/pipeline.yml rename to x-pack/filebeat/module/salesforce/setupaudittrail/ingest/pipeline.yml diff --git a/x-pack/filebeat/module/salesforce/setupaudittrail/manifest.yml b/x-pack/filebeat/module/salesforce/setupaudittrail/manifest.yml new file mode 100644 index 000000000000..39c2f16c2be2 --- /dev/null +++ b/x-pack/filebeat/module/salesforce/setupaudittrail/manifest.yml @@ -0,0 +1,24 @@ +module_version: 1.0 + +var: + - name: input + default: salesforce + - name: tags + default: [salesforce-setupaudittrail, forwarded] + - name: api_version + - name: jwt_enabled + - name: client_id + - name: client_username + - name: client_key_path + - name: jwt_url + - name: client_secret + - name: token_url + - name: user + - name: password + - name: url + - name: real_time + - name: real_time_interval + default: 5m + +ingest_pipeline: ingest/pipeline.yml +input: config/setupaudittrail.yml diff --git a/x-pack/filebeat/module/salesforce/setupaudittrail-rest/test/salesforce_setupaudittrail.ndjson.log b/x-pack/filebeat/module/salesforce/setupaudittrail/test/salesforce_setupaudittrail.ndjson.log similarity index 100% rename from x-pack/filebeat/module/salesforce/setupaudittrail-rest/test/salesforce_setupaudittrail.ndjson.log rename to x-pack/filebeat/module/salesforce/setupaudittrail/test/salesforce_setupaudittrail.ndjson.log diff --git a/x-pack/filebeat/module/salesforce/setupaudittrail-rest/test/salesforce_setupaudittrail.ndjson.log-expected.json b/x-pack/filebeat/module/salesforce/setupaudittrail/test/salesforce_setupaudittrail.ndjson.log-expected.json similarity index 100% rename from x-pack/filebeat/module/salesforce/setupaudittrail-rest/test/salesforce_setupaudittrail.ndjson.log-expected.json rename to x-pack/filebeat/module/salesforce/setupaudittrail/test/salesforce_setupaudittrail.ndjson.log-expected.json