sdd_all: use triple-brace templating (elastic#11286)

The mustache templating system used by ingest pipelines has two levels of
escaping available, not escaped (triple stache) and HTML escaped
(double stache) — see man mustache[1] under "tag types: variables". This can
lead to data corruption, particularly in cases where an operating system has
chosen to use a character requiring escaping in its path syntax.

[1]http://mustache.github.io/mustache.5.html

[git-generate]
for f in $(
	(
		for p in $(
			yq 'select(.owner.github == "elastic/sec-deployment-and-devices")|.name' packages/**/manifest.yml \
			| grep -v -- '---'
		); do
			rg -l -g '*.yml' ": ('\{\{[^{][ .a-zA-Z0-9_]*[^}]}}'|\"\{\{[^{][ .a-zA-Z0-9_]*[^}]}}\")" packages/$p
		done
	)|grep "elasticsearch/ingest_pipeline"|sort|uniq
); do
	sed -i -r "s/: (['\"])\{\{([^{][ .a-zA-Z0-9_]*[^}])}}['\"]/: \1{{{\2}}}\1/g" $f
done
for p in $(git diff --name-only HEAD~1|cut -d/ -f1,2|sort|uniq); do
	(
		cd $p
		elastic-package test pipeline -g
		elastic-package changelog add \
			--description "Use triple-brace Mustache templating when referencing variables in ingest pipelines." \
			--type bugfix \
			--next patch \
			--link elastic#11286
	)>/dev/null 2>&1
done

packages/bluecoat/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.17.3"
+  changes:
+    - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/11286
 - version: "0.17.2"
   changes:
     - description: Changed owners

packages/bluecoat/data_stream/director/elasticsearch/ingest_pipeline/default.yml

@@ -53,7 +53,7 @@ processors:
       ignore_missing: true
   - append:
       field: related.hosts
-      value: '{{host.name}}'
+      value: '{{{host.name}}}'
       allow_duplicates: false
       if: ctx.host?.name != null && ctx.host?.name != ''
   - remove:

packages/bluecoat/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 2.7.0
 name: bluecoat
 title: Blue Coat Director Logs (Deprecated)
-version: "0.17.2"
+version: "0.17.3"
 description: Deprecated. Director is no longer supported.
 categories: ["network", "security", "proxy_security"]
 type: integration

packages/cef/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.17.3"
+  changes:
+    - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/11286
 - version: "2.17.2"
   changes:
     - description: Make dataset name configurable

packages/cef/data_stream/log/elasticsearch/ingest_pipeline/cp-pipeline.yml

@@ -279,8 +279,8 @@ processors:
       field: _tmp_copy
       processor:
         set:
-          field: '{{_ingest._value.to}}'
-          value: '{{_ingest._value.value}}'
+          field: '{{{_ingest._value.to}}}'
+          value: '{{{_ingest._value.value}}}'
   - remove:
       field: _tmp_copy
   - set:

packages/cef/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -57,46 +57,46 @@ processors:
       if: ctx?.cef?.extensions?.fileHash != null && ctx?.cef?.extensions?.fileHash != ''
       field: related.hash
       allow_duplicates: false
-      value: '{{cef.extensions.fileHash}}'
+      value: '{{{cef.extensions.fileHash}}}'
   - append:
       if: ctx?.cef?.extensions?.oldFileHash != null && ctx?.cef?.extensions?.oldFileHash != ''
       field: related.hash
       allow_duplicates: false
-      value: '{{cef.extensions.oldFileHash}}'
+      value: '{{{cef.extensions.oldFileHash}}}'
   - append:
       if: ctx?.destination?.ip != null && ctx?.destination?.ip != ''
       field: related.ip
       allow_duplicates: false
-      value: '{{destination.ip}}'
+      value: '{{{destination.ip}}}'
   - append:
       if: ctx?.destination?.nat?.ip != null && ctx?.destination?.nat?.ip != ''
       field: related.ip
       allow_duplicates: false
-      value: '{{destination.nat.ip}}'
+      value: '{{{destination.nat.ip}}}'
   - append:
       if: ctx?.source?.ip != null && ctx?.source?.ip != ''
       field: related.ip
       allow_duplicates: false
-      value: '{{source.ip}}'
+      value: '{{{source.ip}}}'
   - append:
       if: ctx?.source?.nat?.ip != null && ctx?.source?.nat?.ip != ''
       field: related.ip
       allow_duplicates: false
-      value: '{{source.nat.ip}}'
+      value: '{{{source.nat.ip}}}'
   - append:
       if: ctx?.destination?.user?.name != null
       field: related.user
-      value: '{{destination.user.name}}'
+      value: '{{{destination.user.name}}}'
   - append:
       if: ctx?.source?.user?.name != null && ctx?.source?.user?.name != ''
       field: related.user
       allow_duplicates: false
-      value: '{{source.user.name}}'
+      value: '{{{source.user.name}}}'
   - append:
       if: ctx?.observer?.hostname != null && ctx?.observer?.hostname != ''
       field: related.hosts
       allow_duplicates: false
-      value: '{{observer.hostname}}'
+      value: '{{{observer.hostname}}}'
   - pipeline:
       if: ctx.cef?.device?.vendor == 'FORCEPOINT'
       name: '{{ IngestPipeline "fp-pipeline" }}'
@@ -173,7 +173,7 @@ processors:
       if: ctx._tmp?.observer != null && ctx.observer?.ip == null
       field: observer.ip
       tag: observer append
-      value: '{{_tmp.observer}}'
+      value: '{{{_tmp.observer}}}'
   # Set ECS event outcome from ArcSight outcomes
   - set:
       if: ctx.cef?.extensions?.categoryOutcome == "/Success"

packages/cef/data_stream/log/elasticsearch/ingest_pipeline/fp-pipeline.yml

@@ -5,22 +5,22 @@ processors:
   - set:
       field: rule.id
       ignore_empty_value: true
-      value: '{{cef.extensions.deviceCustomString1}}'
+      value: '{{{cef.extensions.deviceCustomString1}}}'
   # cs2 is natRuleID
   - set:
       field: rule.id
       ignore_empty_value: true
-      value: '{{cef.extensions.deviceCustomString2}}'
+      value: '{{{cef.extensions.deviceCustomString2}}}'
   # cs3 is VulnerabilityReference
   - set:
       field: vulnerability.reference
       ignore_empty_value: true
-      value: '{{cef.extensions.deviceCustomString3}}'
+      value: '{{{cef.extensions.deviceCustomString3}}}'
   # cs4 is virusID
   - set:
       field: cef.forcepoint.virus_id
       ignore_empty_value: true
-      value: '{{cef.extensions.deviceCustomString4}}'
+      value: '{{{cef.extensions.deviceCustomString4}}}'
 on_failure:
   - append:
       field: error.message

packages/cef/manifest.yml

@@ -1,6 +1,6 @@
 name: cef
 title: Common Event Format (CEF)
-version: "2.17.2"
+version: "2.17.3"
 description: Collect logs from CEF Logs with Elastic Agent.
 categories:
   - security

packages/checkpoint/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.34.1"
+  changes:
+    - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/11286
 - version: "1.34.0"
   changes:
     - description: Drop support for EOL OS version R80.X

packages/checkpoint/data_stream/firewall/_dev/test/pipeline/test-checkpoint-with-time.log-expected.json

@@ -208,7 +208,7 @@
                     "zone": "External"
                 },
                 "name": "172.16.2.9",
-                "product": "VPN-1 \\\\u0026 FireWall-1",
+                "product": "VPN-1 \\u0026 FireWall-1",
                 "type": "firewall",
                 "vendor": "Checkpoint"
             },
@@ -298,7 +298,7 @@
                     "zone": "External"
                 },
                 "name": "172.16.2.9",
-                "product": "VPN-1 \\\\u0026 FireWall-1",
+                "product": "VPN-1 \\u0026 FireWall-1",
                 "type": "firewall",
                 "vendor": "Checkpoint"
             },

packages/checkpoint/data_stream/firewall/elasticsearch/ingest_pipeline/default.yml

@@ -192,7 +192,7 @@ processors:
       if: ctx.checkpoint?.type == null
   - set:
       field: observer.product
-      value: "{{checkpoint.product}}"
+      value: "{{{checkpoint.product}}}"
       ignore_empty_value: true
   - rename:
       field: checkpoint.src
@@ -435,37 +435,37 @@ processors:
       if: ctx.checkpoint?.action == 'Failed Log In'
   - append:
       field: related.ip
-      value: "{{source.ip}}"
+      value: "{{{source.ip}}}"
       allow_duplicates: false
       if: ctx.source?.ip != null
   - append:
       field: related.ip
-      value: "{{source.nat.ip}}"
+      value: "{{{source.nat.ip}}}"
       allow_duplicates: false
       if: ctx.source?.nat?.ip != null
   - append:
       field: related.ip
-      value: "{{destination.ip}}"
+      value: "{{{destination.ip}}}"
       allow_duplicates: false
       if: ctx.destination?.ip != null
   - append:
       field: related.ip
-      value: "{{destination.nat.ip}}"
+      value: "{{{destination.nat.ip}}}"
       allow_duplicates: false
       if: ctx.destination?.nat?.ip != null
   - append:
       field: related.hash
-      value: "{{checkpoint.file_md5}}"
+      value: "{{{checkpoint.file_md5}}}"
       allow_duplicates: false
       if: ctx.checkpoint?.file_md5 != null
   - append:
       field: related.hash
-      value: "{{checkpoint.file_sha1}}"
+      value: "{{{checkpoint.file_sha1}}}"
       allow_duplicates: false
       if: ctx.checkpoint?.file_sha1 != null
   - append:
       field: related.hash
-      value: "{{checkpoint.file_sha256}}"
+      value: "{{{checkpoint.file_sha256}}}"
       allow_duplicates: false
       if: ctx.checkpoint?.file_sha256 != null
   - rename:
@@ -1097,22 +1097,22 @@ processors:
       if: ctx.checkpoint?.sys_message != null
   - append:
       field: related.user
-      value: "{{checkpoint.user}}"
+      value: "{{{checkpoint.user}}}"
       allow_duplicates: false
       if: ctx.checkpoint?.user != null
   - append:
       field: related.user
-      value: "{{checkpoint.administrator}}"
+      value: "{{{checkpoint.administrator}}}"
       allow_duplicates: false
       if: ctx.checkpoint?.administrator != null
   - append:
       field: related.user
-      value: "{{checkpoint.src_user_name}}"
+      value: "{{{checkpoint.src_user_name}}}"
       allow_duplicates: false
       if: ctx.checkpoint?.src_user_name != null
   - append:
       field: related.user
-      value: "{{checkpoint.dst_user_name}}"
+      value: "{{{checkpoint.dst_user_name}}}"
       allow_duplicates: false
       if: ctx.checkpoint?.dst_user_name != null
   - script:
@@ -1303,4 +1303,4 @@ on_failure:
       value: pipeline_error
   - append:
       field: error.message
-      value: "{{ _ingest.on_failure_message }}"
+      value: "{{{ _ingest.on_failure_message }}}"

packages/checkpoint/manifest.yml

@@ -1,6 +1,6 @@
 name: checkpoint
 title: Check Point
-version: "1.34.0"
+version: "1.34.1"
 description: Collect logs from Check Point with Elastic Agent.
 type: integration
 format_version: "3.0.3"

packages/cisco_ise/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.23.1"
+  changes:
+    - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/11286
 - version: "1.23.0"
   changes:
     - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."

packages/cisco_ise/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -158,7 +158,7 @@ processors:
       on_failure:
         - append:
             field: error.message
-            value: "{{ _ingest.on_failure_message }}"
+            value: "{{{ _ingest.on_failure_message }}}"
   - remove:
       field: cisco_ise.log.log_details.ConfigVersionId
       ignore_missing: true

packages/cisco_ise/data_stream/log/elasticsearch/ingest_pipeline/pipeline_administrative_and_operational_audit.yml

@@ -348,7 +348,7 @@ processors:
       on_failure:
         - append:
             field: error.message
-            value: "{{ _ingest.on_failure_message }}"
+            value: "{{{ _ingest.on_failure_message }}}"
   - remove:
       field: cisco_ise.log.log_details.DestinationPort
       ignore_missing: true
@@ -365,7 +365,7 @@ processors:
       on_failure:
         - append:
             field: error.message
-            value: "{{ _ingest.on_failure_message }}"
+            value: "{{{ _ingest.on_failure_message }}}"
   - remove:
       field: cisco_ise.log.log_details.FailureFlag
       ignore_missing: true
@@ -423,7 +423,7 @@ processors:
       on_failure:
         - append:
             field: error.message
-            value: "{{ _ingest.on_failure_message }}"
+            value: "{{{ _ingest.on_failure_message }}}"
   - remove:
       field: cisco_ise.log.log_details.FeedServiceQueryToTime
       ignore_missing: true
@@ -436,7 +436,7 @@ processors:
       on_failure:
         - append:
             field: error.message
-            value: "{{ _ingest.on_failure_message }}"
+            value: "{{{ _ingest.on_failure_message }}}"
   - remove:
       field: cisco_ise.log.log_details.FeedServiceQueryFromTime
       ignore_missing: true
@@ -512,7 +512,7 @@ processors:
       on_failure:
         - append:
             field: error.message
-            value: "{{ _ingest.on_failure_message }}"
+            value: "{{{ _ingest.on_failure_message }}}"
   - remove:
       field: cisco_ise.log.log_details.ResponseTime
       ignore_missing: true

packages/cisco_ise/data_stream/log/elasticsearch/ingest_pipeline/pipeline_policy_diagnostics.yml

@@ -100,7 +100,7 @@ processors:
       on_failure:
         - append:
             field: error.message
-            value: "{{ _ingest.on_failure_message }}"
+            value: "{{{ _ingest.on_failure_message }}}"
   - remove:
       field: cisco_ise.log.log_details.RequestReceivedTime
       ignore_missing: true

packages/cisco_ise/data_stream/log/elasticsearch/ingest_pipeline/pipeline_posture_and_client_provisioning_audit.yml

@@ -50,7 +50,7 @@ processors:
             ignore_missing: true
         - append:
             field: error.message
-            value: "{{ _ingest.on_failure_message }}"
+            value: "{{{ _ingest.on_failure_message }}}"
   - kv:
       field: cisco_ise.log.log_details_raw
       target_field: cisco_ise.log.log_details

packages/cisco_ise/data_stream/log/elasticsearch/ingest_pipeline/pipeline_radius_accounting.yml

@@ -52,7 +52,7 @@ processors:
             ignore_missing: true
         - append:
             field: error.message
-            value: "{{ _ingest.on_failure_message }}"
+            value: "{{{ _ingest.on_failure_message }}}"
   - kv:
       field: cisco_ise.log.log_details_raw
       target_field: cisco_ise.log.log_details
@@ -221,7 +221,7 @@ processors:
       on_failure:
         - append:
             field: error.message
-            value: "{{ _ingest.on_failure_message }}"
+            value: "{{{ _ingest.on_failure_message }}}"
   - remove:
       field: cisco_ise.log.log_details.Event-Timestamp
       ignore_missing: true

packages/cisco_ise/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.3"
 name: cisco_ise
 title: Cisco ISE
-version: "1.23.0"
+version: "1.23.1"
 description: Collect logs from Cisco ISE with Elastic Agent.
 type: integration
 categories:

packages/citrix_waf/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.16.1"
+  changes:
+    - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/11286
 - version: "1.16.0"
   changes:
     - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."

packages/citrix_waf/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -153,4 +153,4 @@ on_failure:
       value: pipeline_error
   - append:
       field: error.message
-      value: "{{ _ingest.on_failure_message }}"
+      value: "{{{ _ingest.on_failure_message }}}"