ssi_all: use triple-brace templating (elastic#11284)

The mustache templating system used by ingest pipelines has two levels of escaping available, not escaped (triple stache) and HTML escaped (double stache) — see man mustache[1] under "tag types: variables". This can lead to data corruption, particularly in cases where an operating system has chosen to use a character requiring escaping in its path syntax. The cloudflare package is omitted from this set of changes due to ci difficulties with its system tests. [1]http://mustache.github.io/mustache.5.html [git-generate] for f in $( ( for p in $( yq 'select(.owner.github == "elastic/security-service-integrations")|.name' packages/**/manifest.yml \ | grep -v -- '---' ); do rg -l -g '*.yml' ": ('\{\{[^{][ .a-zA-Z0-9_]*[^}]}}'|\"\{\{[^{][ .a-zA-Z0-9_]*[^}]}}\")" packages/$p done )|grep -v "cloudflare"|grep "elasticsearch/ingest_pipeline"|sort|uniq ); do sed -i -r "s/: (['\"])\{\{([^{][ .a-zA-Z0-9_]*[^}])}}['\"]/: \1{{{\2}}}\1/g" $f done for p in $(git diff --name-only HEAD~1|cut -d/ -f1,2|sort|uniq); do ( cd $p elastic-package test pipeline -g elastic-package changelog add \ --description "Use triple-brace Mustache templating when referencing variables in ingest pipelines." \ --type bugfix \ --next patch \ --link elastic#11284 )>/dev/null 2>&1 done
taylor-swanson · Oct 1, 2024 · be59434 · be59434
1 parent 022bc30
commit be59434
Show file tree

Hide file tree

Showing 222 changed files with 696 additions and 435 deletions.
diff --git a/packages/1password/changelog.yml b/packages/1password/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.30.1"
+  changes:
+    - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/11284
 - version: "1.30.0"
   changes:
     - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."

diff --git a/packages/1password/data_stream/item_usages/elasticsearch/ingest_pipeline/default.yml b/packages/1password/data_stream/item_usages/elasticsearch/ingest_pipeline/default.yml
@@ -40,22 +40,22 @@ processors:
   #########################
   - append:
       field: related.user
-      value: "{{onepassword.user.uuid}}"
+      value: "{{{onepassword.user.uuid}}}"
       allow_duplicates: false
       if: ctx?.onepassword?.user?.uuid != null
   - append:
       field: related.user
-      value: "{{onepassword.user.email}}"
+      value: "{{{onepassword.user.email}}}"
       allow_duplicates: false
       if: ctx?.onepassword?.user?.email != null
   - append:
       field: related.user
-      value: "{{onepassword.user.name}}"
+      value: "{{{onepassword.user.name}}}"
       allow_duplicates: false
       if: ctx?.onepassword?.user?.name != null
   - append:
       field: related.ip
-      value: "{{onepassword.client.ip_address}}"
+      value: "{{{onepassword.client.ip_address}}}"
       allow_duplicates: false
       if: ctx?.onepassword?.client?.ip_address != null
   ######################

diff --git a/packages/1password/data_stream/signin_attempts/elasticsearch/ingest_pipeline/default.yml b/packages/1password/data_stream/signin_attempts/elasticsearch/ingest_pipeline/default.yml
@@ -50,22 +50,22 @@ processors:
   #########################
   - append:
       field: related.user
-      value: "{{onepassword.target_user.uuid}}"
+      value: "{{{onepassword.target_user.uuid}}}"
       allow_duplicates: false
       if: ctx?.onepassword?.target_user?.uuid != null
   - append:
       field: related.user
-      value: "{{onepassword.target_user.email}}"
+      value: "{{{onepassword.target_user.email}}}"
       allow_duplicates: false
       if: ctx?.onepassword?.target_user?.email != null
   - append:
       field: related.user
-      value: "{{onepassword.target_user.name}}"
+      value: "{{{onepassword.target_user.name}}}"
       allow_duplicates: false
       if: ctx?.onepassword?.target_user?.name != null
   - append:
       field: related.ip
-      value: "{{onepassword.client.ip_address}}"
+      value: "{{{onepassword.client.ip_address}}}"
       allow_duplicates: false
       if: ctx?.onepassword?.client?.ip_address != null
   ######################

diff --git a/packages/1password/manifest.yml b/packages/1password/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: 1password
 title: "1Password"
-version: "1.30.0"
+version: "1.30.1"
 description: Collect logs from 1Password with Elastic Agent.
 type: integration
 categories:

diff --git a/packages/akamai/changelog.yml b/packages/akamai/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.25.2"
+  changes:
+    - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/11284
 - version: "2.25.1"
   changes:
     - description: Fix definition of subfields of nested objects

diff --git a/packages/akamai/data_stream/siem/elasticsearch/ingest_pipeline/default.yml b/packages/akamai/data_stream/siem/elasticsearch/ingest_pipeline/default.yml
@@ -426,7 +426,7 @@ processors:
   ##
   - append:
       field: related.ip
-      value: "{{source.ip}}"
+      value: "{{{source.ip}}}"
       allow_duplicates: false
   - set:
       field: client

diff --git a/packages/akamai/manifest.yml b/packages/akamai/manifest.yml
@@ -1,6 +1,6 @@
 name: akamai
 title: Akamai
-version: "2.25.1"
+version: "2.25.2"
 description: Collect logs from Akamai with Elastic Agent.
 type: integration
 format_version: "3.0.2"

diff --git a/packages/atlassian_bitbucket/changelog.yml b/packages/atlassian_bitbucket/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.2.1"
+  changes:
+    - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/11284
 - version: "2.2.0"
   changes:
     - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."

diff --git a/packages/atlassian_bitbucket/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/atlassian_bitbucket/data_stream/audit/elasticsearch/ingest_pipeline/default.yml
@@ -386,27 +386,27 @@ processors:
         }
   - append:
       field: related.user
-      value: '{{user.name}}'
+      value: '{{{user.name}}}'
       allow_duplicates: false
       if: ctx.user?.name != null
   - append:
       field: related.user
-      value: '{{user.target.name}}'
+      value: '{{{user.target.name}}}'
       allow_duplicates: false
       if: ctx.user?.target?.name != null
   - append:
       field: related.user
-      value: '{{user.changes.name}}'
+      value: '{{{user.changes.name}}}'
       allow_duplicates: false
       if: ctx.user?.changes?.name != null
   - append:
       field: related.ip
-      value: '{{source.ip}}'
+      value: '{{{source.ip}}}'
       allow_duplicates: false
       if: ctx.source?.ip != null
   - append:
       field: related.hosts
-      value: '{{_tmp.service.domain}}'
+      value: '{{{_tmp.service.domain}}}'
       allow_duplicates: false
       if: ctx._tmp?.service?.domain != null
   - remove:

diff --git a/packages/atlassian_bitbucket/manifest.yml b/packages/atlassian_bitbucket/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: atlassian_bitbucket
 title: Atlassian Bitbucket
-version: "2.2.0"
+version: "2.2.1"
 description: Collect logs from Atlassian Bitbucket with Elastic Agent.
 type: integration
 categories:

diff --git a/packages/atlassian_confluence/changelog.yml b/packages/atlassian_confluence/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.26.1"
+  changes:
+    - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/11284
 - version: "1.26.0"
   changes:
     - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."

diff --git a/packages/atlassian_confluence/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/atlassian_confluence/data_stream/audit/elasticsearch/ingest_pipeline/default.yml
@@ -376,22 +376,22 @@ processors:
         }
   - append:
       field: related.user
-      value: '{{user.name}}'
+      value: '{{{user.name}}}'
       allow_duplicates: false
       if: ctx.user?.name != null
   - append:
       field: related.user
-      value: '{{user.target.name}}'
+      value: '{{{user.target.name}}}'
       allow_duplicates: false
       if: ctx.user?.target?.name != null
   - append:
       field: related.user
-      value: '{{user.changes.name}}'
+      value: '{{{user.changes.name}}}'
       allow_duplicates: false
       if: ctx.user?.changes?.name != null
   - append:
       field: related.ip
-      value: '{{source.ip}}'
+      value: '{{{source.ip}}}'
       allow_duplicates: false
       if: ctx.source?.ip != null
   - remove:

diff --git a/...ages/atlassian_confluence/data_stream/audit/elasticsearch/ingest_pipeline/self-hosted.yml b/...ages/atlassian_confluence/data_stream/audit/elasticsearch/ingest_pipeline/self-hosted.yml
@@ -74,7 +74,7 @@ processors:
     ignore_empty_value: true
 - append:
     field: related.hosts
-    value: '{{_tmp.service.domain}}'
+    value: '{{{_tmp.service.domain}}}'
     allow_duplicates: false
     if: ctx._tmp?.service?.domain != null
 on_failure:

diff --git a/packages/atlassian_confluence/manifest.yml b/packages/atlassian_confluence/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: atlassian_confluence
 title: Atlassian Confluence
-version: "1.26.0"
+version: "1.26.1"
 description: Collect logs from Atlassian Confluence with Elastic Agent.
 type: integration
 categories:

diff --git a/packages/atlassian_jira/changelog.yml b/packages/atlassian_jira/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.27.1"
+  changes:
+    - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/11284
 - version: "1.27.0"
   changes:
     - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."

diff --git a/packages/atlassian_jira/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/atlassian_jira/data_stream/audit/elasticsearch/ingest_pipeline/default.yml
@@ -355,27 +355,27 @@ processors:
         }
   - append:
       field: related.user
-      value: '{{user.name}}'
+      value: '{{{user.name}}}'
       allow_duplicates: false
       if: ctx.user?.name != null
   - append:
       field: related.user
-      value: '{{user.target.name}}'
+      value: '{{{user.target.name}}}'
       allow_duplicates: false
       if: ctx.user?.target?.name != null
   - append:
       field: related.user
-      value: '{{user.changes.name}}'
+      value: '{{{user.changes.name}}}'
       allow_duplicates: false
       if: ctx.user?.changes?.name != null
   - append:
       field: related.ip
-      value: '{{source.ip}}'
+      value: '{{{source.ip}}}'
       allow_duplicates: false
       if: ctx.source?.ip != null
   - append:
       field: related.hosts
-      value: '{{_tmp.service.domain}}'
+      value: '{{{_tmp.service.domain}}}'
       allow_duplicates: false
       if: ctx._tmp?.service?.domain != null
   - remove:

diff --git a/packages/atlassian_jira/data_stream/audit/elasticsearch/ingest_pipeline/self-hosted.yml b/packages/atlassian_jira/data_stream/audit/elasticsearch/ingest_pipeline/self-hosted.yml
@@ -85,7 +85,7 @@ processors:
     ignore_empty_value: true
 - append:
     field: related.hosts
-    value: '{{_tmp.service.domain}}'
+    value: '{{{_tmp.service.domain}}}'
     allow_duplicates: false
     if: ctx._tmp?.service?.domain != null
 on_failure:

diff --git a/packages/atlassian_jira/manifest.yml b/packages/atlassian_jira/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: atlassian_jira
 title: Atlassian Jira
-version: "1.27.0"
+version: "1.27.1"
 description: Collect logs from Atlassian Jira with Elastic Agent.
 type: integration
 categories:

diff --git a/packages/azure_frontdoor/changelog.yml b/packages/azure_frontdoor/changelog.yml
@@ -1,3 +1,8 @@
+- version: "2.0.1"
+  changes:
+    - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/11284
 - version: "2.0.0"
   changes:
     - description: "Changed keyword field type to double in time_to_first_byte, time_taken fields and to date in time fields"

diff --git a/packages/azure_frontdoor/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/packages/azure_frontdoor/data_stream/access/elasticsearch/ingest_pipeline/default.yml
@@ -241,13 +241,13 @@ processors:
   # set user.email to the original name if the above grok succeeded.
   - set:
       field: user.email
-      value: '{{azure.frontdoor.access.identity.claims_initiated_by_user.name}}'
+      value: '{{{azure.frontdoor.access.identity.claims_initiated_by_user.name}}}'
       ignore_empty_value: true
       if: 'ctx.user?.name != null'
   # set user.name to the original name if the above grok failed (name format is not an email).
   - set:
       field: user.name
-      value: '{{azure.frontdoor.access.identity.claims_initiated_by_user.name}}'
+      value: '{{{azure.frontdoor.access.identity.claims_initiated_by_user.name}}}'
       ignore_empty_value: true
       if: 'ctx.user?.name == null'
   - rename:
@@ -260,22 +260,22 @@ processors:
       ignore_missing: true
   - append:
       field: user.roles
-      value: '{{azure.frontdoor.access.identity.authorization.evidence.role}}'
+      value: '{{{azure.frontdoor.access.identity.authorization.evidence.role}}}'
       allow_duplicates: false
       if: ctx.azure?.frontdoor?.access?.identity?.authorization?.evidence?.role != null
   - append:
       field: related.user
-      value: '{{user.name}}'
+      value: '{{{user.name}}}'
       allow_duplicates: false
       if: 'ctx.user?.name != null'
   - append:
       field: related.user
-      value: '{{user.full_name}}'
+      value: '{{{user.full_name}}}'
       allow_duplicates: false
       if: 'ctx.user?.name != null'
   - append:
       field: related.user
-      value: '{{user.id}}'
+      value: '{{{user.id}}}'
       allow_duplicates: false
       if: 'ctx.user?.name != null'
   - rename:

diff --git a/packages/azure_frontdoor/data_stream/waf/elasticsearch/ingest_pipeline/default.yml b/packages/azure_frontdoor/data_stream/waf/elasticsearch/ingest_pipeline/default.yml
@@ -184,13 +184,13 @@ processors:
   # set user.email to the original name if the above grok succeeded.
   - set:
       field: user.email
-      value: '{{azure.frontdoor.waf.identity.claims_initiated_by_user.name}}'
+      value: '{{{azure.frontdoor.waf.identity.claims_initiated_by_user.name}}}'
       ignore_empty_value: true
       if: 'ctx.user?.name != null'
   # set user.name to the original name if the above grok failed (name format is not an email).
   - set:
       field: user.name
-      value: '{{azure.frontdoor.waf.identity.claims_initiated_by_user.name}}'
+      value: '{{{azure.frontdoor.waf.identity.claims_initiated_by_user.name}}}'
       ignore_empty_value: true
       if: 'ctx.user?.name == null'
   - rename:
@@ -203,22 +203,22 @@ processors:
       ignore_missing: true
   - append:
       field: user.roles
-      value: '{{azure.frontdoor.waf.identity.authorization.evidence.role}}'
+      value: '{{{azure.frontdoor.waf.identity.authorization.evidence.role}}}'
       allow_duplicates: false
       if: ctx.azure?.frontdoor?.waf?.identity?.authorization?.evidence?.role != null
   - append:
       field: related.user
-      value: '{{user.name}}'
+      value: '{{{user.name}}}'
       allow_duplicates: false
       if: 'ctx.user?.name != null'
   - append:
       field: related.user
-      value: '{{user.full_name}}'
+      value: '{{{user.full_name}}}'
       allow_duplicates: false
       if: 'ctx.user?.name != null'
   - append:
       field: related.user
-      value: '{{user.id}}'
+      value: '{{{user.id}}}'
       allow_duplicates: false
       if: 'ctx.user?.name != null'
   - rename:

diff --git a/packages/azure_frontdoor/manifest.yml b/packages/azure_frontdoor/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: azure_frontdoor
 title: "Azure Frontdoor"
-version: "2.0.0"
+version: "2.0.1"
 description: "This Elastic integration collects logs from Azure Frontdoor."
 type: integration
 categories:

diff --git a/packages/barracuda/changelog.yml b/packages/barracuda/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.15.1"
+  changes:
+    - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/11284
 - version: "1.15.0"
   changes:
     - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."