Merge branch 'main' into environment-proxy-fix

.buildkite/bk.integration.pipeline.yml

@@ -28,7 +28,7 @@ steps:
     steps:
       - label: "Win2022:sudo:{{matrix}}"
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-windows' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
           .buildkite/scripts/integration-tests.ps1 {{matrix}} true
         artifact_paths:
           - build/**
@@ -45,7 +45,7 @@ steps:
 
       - label: "Win2022:non-sudo:{{matrix}}"
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-windows' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
           .buildkite/scripts/integration-tests.ps1 {{matrix}} false
         artifact_paths:
           - build/**
@@ -63,8 +63,9 @@ steps:
       - integration-ess
     steps:
       - label: "x86_64:non-sudo: {{matrix}}"
+        # only packaging-ubuntu-x86-64 artifact dependency is required
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
         artifact_paths:
           - build/**
@@ -77,8 +78,9 @@ steps:
           - default
 
       - label: "x86_64:sudo: {{matrix}}"
+        # due to deb group present in matrix tar.gz and deb packages artifacts are required
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** . --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
         artifact_paths:
           - build/**
@@ -102,7 +104,7 @@ steps:
       - label: "arm:sudo: {{matrix}}"
         skip: true
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
         artifact_paths:
           - build/**
@@ -126,7 +128,7 @@ steps:
       - label: "arm:non-sudo: {{matrix}}"
         skip: true
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
         artifact_paths:
           - build/**
@@ -145,7 +147,7 @@ steps:
     steps:
       - label: "x86_64:sudo:rpm"
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** . --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
           .buildkite/scripts/steps/integration_tests_tf.sh rpm true
         artifact_paths:
           - build/**

.buildkite/integration.pipeline.yml

@@ -5,80 +5,131 @@ env:
   VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
 
 steps:
-  - label: "Integration tests: packaging"
-    key: "package-it"
-    command: ".buildkite/scripts/steps/integration-package.sh"
-    artifact_paths:
-      - build/distributions/**
-    agents:
-      provider: "gcp"
-      machineType: "n1-standard-8"
+  - group: "Integration tests: packaging"
+    key: "int-packaging"
+    steps:
+      - label: "Packaging: Ubuntu x86_64"
+        key: "packaging-ubuntu-x86-64"
+        env:
+          PACKAGES: "tar.gz"
+          PLATFORMS: "linux/amd64"
+        command: ".buildkite/scripts/steps/integration-package.sh"
+        artifact_paths:
+        - build/distributions/**
+        agents:
+          provider: "gcp"
+          machineType: "n2-standard-8"
+
+      - label: "Packaging: Ubuntu arm64"
+        key: "packaging-ubuntu-arm64"
+        env:
+          PACKAGES: "tar.gz"
+          PLATFORMS: "linux/arm64"
+        command: ".buildkite/scripts/steps/integration-package.sh"
+        artifact_paths:
+          - build/distributions/**
+        agents:
+          provider: "gcp"
+          machineType: "n2-standard-8"
+
+      - label: "Packaging: Windows"
+        key: "packaging-windows"
+        env:
+          PACKAGES: "zip"
+          PLATFORMS: "windows/amd64"
+        command: ".buildkite/scripts/steps/integration-package.sh"
+        artifact_paths:
+          - build/distributions/**
+        agents:
+          provider: "gcp"
+          machineType: "n2-standard-8"
+
+      - label: "Packaging: Containers {{matrix.ext}} {{matrix.arch}}"
+        key: "packaging-containers"
+        env:
+          PACKAGES: "{{matrix.ext}}"
+          PLATFORMS: "{{matrix.arch}}"
+        command: ".buildkite/scripts/steps/integration-package.sh"
+        artifact_paths:
+          - build/distributions/**
+        agents:
+          provider: "gcp"
+          machineType: "n2-standard-8"
+        matrix:
+          setup:
+            arch:
+              - linux/amd64
+              - linux/arm64
+            ext:
+              - rpm
+              - deb
 
   - label: "Serverless integration test"
     key: "serverless-integration-tests"
     depends_on:
-      - package-it
+      - int-packaging
     concurrency_group: elastic-agent-extended-testing/serverless-integration
     concurrency: 8
     env:
       # we run each step in a different data center to spread the load
       TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
     command: |
-      buildkite-agent artifact download build/distributions/** . --step 'package-it'
+      buildkite-agent artifact download "build/distributions/**" . $BUILDKITE_BUILD_ID
       .buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite
     artifact_paths:
       - "build/TEST-**"
       - "build/diagnostics/*"
     agents:
       provider: "gcp"
-      machineType: "n1-standard-8"
+      machineType: "n2-standard-8"
     notify:
       - github_commit_status:
           context: "buildkite/elastic-agent-extended-testing - Serverless integration test"
 
   - label: "Extended runtime leak tests"
     key: "extended-integration-tests"
     depends_on:
-      - package-it
+      - int-packaging
     concurrency_group: elastic-agent-extended-testing/leak-tests
     concurrency: 8
     env:
       TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-b"
     command: |
-      buildkite-agent artifact download build/distributions/** . --step 'package-it'
+      buildkite-agent artifact download "build/distributions/**" . $BUILDKITE_BUILD_ID
       .buildkite/scripts/steps/integration_tests.sh stateful integration:TestForResourceLeaks
     artifact_paths:
       - "build/TEST-**"
       - "build/diagnostics/*"
     agents:
       provider: "gcp"
-      machineType: "n1-standard-8"
+      machineType: "n2-standard-8"
     notify:
       - github_commit_status:
           context: "buildkite/elastic-agent-extended-testing - Extended runtime leak tests"
 
   - label: "Triggering Integration tests"
     depends_on:
-      - package-it
+      - int-packaging
     trigger: "elastic-agent-extended-testing-bk"
     build:
       commit: "${BUILDKITE_COMMIT}"
       branch: "${BUILDKITE_BRANCH}"
 
   - label: "Serverless Beats Tests"
+    # To speedup the build process only packaging-ubuntu-x86-64 artifact dependency is required
     depends_on:
-      - package-it
+      - packaging-ubuntu-x86-64
     key: "serverless-beats-integration-tests"
     concurrency_group: elastic-agent-extended-testing/beats-integration
     concurrency: 8
     env:
       TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
     command: |
-      buildkite-agent artifact download build/distributions/** . --step 'package-it'
+      buildkite-agent artifact download "build/distributions/**" . $BUILDKITE_BUILD_ID
       .buildkite/scripts/steps/beats_tests.sh
     agents:
       provider: "gcp"
-      machineType: "n1-standard-8"
+      machineType: "n2-standard-8"
     notify:
       - github_commit_status:
           context: "buildkite/elastic-agent-extended-testing - Serverless Beats Tests"

.buildkite/pull-requests.json

@@ -13,7 +13,7 @@
             "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it|extended))|^/test\\W*(?:extended|)",
             "skip_ci_labels": [ "skip-ci" ],
             "skip_target_branches": [ ],
-            "skip_ci_on_only_changed": [ "^.ci/", "^.github/", "^updatecli-compose.yaml", "^changelog", "^docs/", "\\.md$", "^docker-compose.yml", "^.pre-commit-config.yaml", "skaffold.yaml", "^Dockerfile.skaffold", "^Dockerfile"],
+            "skip_ci_on_only_changed": [ "^.ci/", "^.github/", "^updatecli-compose.yaml", "^changelog", "^docs/", "\\.md$", "^docker-compose.yml", "^.mergify.yml", "^.pre-commit-config.yaml", "skaffold.yaml", "^Dockerfile.skaffold", "^Dockerfile"],
             "always_require_ci_on_changed": [ ]
         },
         {

.buildkite/scripts/steps/integration-package.sh

@@ -2,5 +2,11 @@
 set -euo pipefail
 
 source .buildkite/scripts/common.sh
+
 # Remove AGENT_PACKAGE_VERSION pinning as soon as 9.0.0 is released
-AGENT_PACKAGE_VERSION=9.0.0 PACKAGES=tar.gz,zip,rpm,deb PLATFORMS=linux/amd64,linux/arm64,windows/amd64  SNAPSHOT=true EXTERNAL=true  mage package
+export AGENT_PACKAGE_VERSION=9.0.0
+
+export SNAPSHOT="true"
+export EXTERNAL="true"
+
+mage package

.mergify.yml

@@ -374,6 +374,19 @@ pull_request_rules:
         labels:
           - "backport"
         title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
+  - name: backport patches to 9.0 branch
+    conditions:
+      - merged
+      - label=backport-9.0
+    actions:
+      backport:
+        assignees:
+          - "{{ author }}"
+        branches:
+          - "9.0"
+        labels:
+          - "backport"
+        title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
 
   - name: backport patches to all active minor branches for the 8 major.
     conditions:
@@ -386,6 +399,7 @@ pull_request_rules:
         # NOTE: this list needs to be changed when a new minor branch is created
         #       or an existing minor branch reached EOL.
         branches:
+          - "8.x"
           - "8.18"
           - "8.17"
           - "8.16"

changelog/fragments/1738874791-Fix-secret_paths-redaction-along-complex-paths.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# Change summary; a 80ish characters long description of the change.
+summary: Fix secret_paths redaction along complex paths
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/6710
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234

internal/pkg/diagnostics/diagnostics.go

@@ -398,7 +398,8 @@ func redactKey(k string) bool {
 		strings.Contains(k, "passphrase") ||
 		strings.Contains(k, "password") ||
 		strings.Contains(k, "token") ||
-		strings.Contains(k, "key")
+		strings.Contains(k, "key") ||
+		strings.Contains(k, "secret")
 }
 
 func zipLogs(zw *zip.Writer, ts time.Time, topPath string, excludeEvents bool) error {
@@ -593,19 +594,23 @@ func RedactSecretPaths(mapStr map[string]any, errOut io.Writer) map[string]any {
 		fmt.Fprintln(errOut, "No output redaction: secret_paths attribute is not a list.")
 		return mapStr
 	}
-	cfg := ucfg.MustNewFrom(mapStr)
+	cfg := ucfg.MustNewFrom(mapStr, ucfg.PathSep("."))
 	for _, v := range arr {
 		key, ok := v.(string)
 		if !ok {
 			fmt.Fprintf(errOut, "No output redaction for %q: expected type string, is type %T.\n", v, v)
 			continue
 		}
 
-		if ok, _ := cfg.Has(key, -1, ucfg.PathSep(".")); ok {
+		if ok, err := cfg.Has(key, -1, ucfg.PathSep(".")); err != nil {
+			fmt.Fprintf(errOut, "Error redacting secret path %q: %v.\n", key, err)
+		} else if ok {
 			err := cfg.SetString(key, -1, REDACTED, ucfg.PathSep("."))
 			if err != nil {
 				fmt.Fprintf(errOut, "No output redaction for %q: %v.\n", key, err)
 			}
+		} else {
+			fmt.Fprintf(errOut, "Unable to find secret path %q for redaction.\n", key)
 		}
 	}
 	result, err := config.MustNewConfigFrom(cfg).ToMapStr()

internal/pkg/diagnostics/diagnostics_test.go

@@ -248,16 +248,55 @@ secret_paths:
     - inputs.0.redactKey
     - inputs.1.missingKey
     - outputs.default.redactOtherKey
+`,
+	}, {
+		name: "path in nested list",
+		input: []byte(`id: test-policy
+inputs:
+    - type: httpjson
+      data_stream:
+        namespace: default
+      streams:
+        - config_version: "2"
+          request.transforms:
+            - set:
+                target: header.Authorization
+                value: SSWS this-should-be-redacted
+            - set:
+                target: url.params.limit
+                value: "1000"
+secret_paths:
+    - inputs.0.streams.0.request.transforms.0.set.value
+`),
+		expect: `id: test-policy
+inputs:
+    - data_stream:
+        namespace: default
+      streams:
+        - config_version: "2"
+          request:
+            transforms:
+                - set:
+                    target: header.Authorization
+                    value: <REDACTED>
+                - set:
+                    target: url.params.limit
+                    value: "1000"
+      type: httpjson
+secret_paths:
+    - inputs.0.streams.0.request.transforms.0.set.value
 `,
 	}}
 
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
 			file := client.DiagnosticFileResult{Content: tc.input, ContentType: "application/yaml"}
 			var out bytes.Buffer
-			err := writeRedacted(io.Discard, &out, "testPath", file)
+			var errOut bytes.Buffer
+			err := writeRedacted(&errOut, &out, "testPath", file)
 			require.NoError(t, err)
 
+			t.Logf("Error output: %s", errOut.String())
 			assert.Equal(t, tc.expect, out.String())
 		})
 	}