diff --git a/.buildkite/bk.integration.pipeline.yml b/.buildkite/bk.integration.pipeline.yml index 6d61b8743fb..178830ce575 100644 --- a/.buildkite/bk.integration.pipeline.yml +++ b/.buildkite/bk.integration.pipeline.yml @@ -28,7 +28,7 @@ steps: steps: - label: "Win2022:sudo:{{matrix}}" command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} + buildkite-agent artifact download build/distributions/** . --step 'packaging-windows' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} .buildkite/scripts/integration-tests.ps1 {{matrix}} true artifact_paths: - build/** @@ -45,7 +45,7 @@ steps: - label: "Win2022:non-sudo:{{matrix}}" command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} + buildkite-agent artifact download build/distributions/** . --step 'packaging-windows' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} .buildkite/scripts/integration-tests.ps1 {{matrix}} false artifact_paths: - build/** @@ -63,8 +63,9 @@ steps: - integration-ess steps: - label: "x86_64:non-sudo: {{matrix}}" + # only packaging-ubuntu-x86-64 artifact dependency is required command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} + buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false artifact_paths: - build/** @@ -77,8 +78,9 @@ steps: - default - label: "x86_64:sudo: {{matrix}}" + # due to deb group present in matrix tar.gz and deb packages artifacts are required command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} + buildkite-agent artifact download build/distributions/** . --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true artifact_paths: - build/** @@ -102,7 +104,7 @@ steps: - label: "arm:sudo: {{matrix}}" skip: true command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} + buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true artifact_paths: - build/** @@ -126,7 +128,7 @@ steps: - label: "arm:non-sudo: {{matrix}}" skip: true command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} + buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false artifact_paths: - build/** @@ -145,7 +147,7 @@ steps: steps: - label: "x86_64:sudo:rpm" command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} + buildkite-agent artifact download build/distributions/** . --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} .buildkite/scripts/steps/integration_tests_tf.sh rpm true artifact_paths: - build/** diff --git a/.buildkite/integration.pipeline.yml b/.buildkite/integration.pipeline.yml index 3095024feff..45434e62fc8 100644 --- a/.buildkite/integration.pipeline.yml +++ b/.buildkite/integration.pipeline.yml @@ -5,33 +5,83 @@ env: VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp" steps: - - label: "Integration tests: packaging" - key: "package-it" - command: ".buildkite/scripts/steps/integration-package.sh" - artifact_paths: - - build/distributions/** - agents: - provider: "gcp" - machineType: "n1-standard-8" + - group: "Integration tests: packaging" + key: "int-packaging" + steps: + - label: "Packaging: Ubuntu x86_64" + key: "packaging-ubuntu-x86-64" + env: + PACKAGES: "tar.gz" + PLATFORMS: "linux/amd64" + command: ".buildkite/scripts/steps/integration-package.sh" + artifact_paths: + - build/distributions/** + agents: + provider: "gcp" + machineType: "n2-standard-8" + + - label: "Packaging: Ubuntu arm64" + key: "packaging-ubuntu-arm64" + env: + PACKAGES: "tar.gz" + PLATFORMS: "linux/arm64" + command: ".buildkite/scripts/steps/integration-package.sh" + artifact_paths: + - build/distributions/** + agents: + provider: "gcp" + machineType: "n2-standard-8" + + - label: "Packaging: Windows" + key: "packaging-windows" + env: + PACKAGES: "zip" + PLATFORMS: "windows/amd64" + command: ".buildkite/scripts/steps/integration-package.sh" + artifact_paths: + - build/distributions/** + agents: + provider: "gcp" + machineType: "n2-standard-8" + + - label: "Packaging: Containers {{matrix.ext}} {{matrix.arch}}" + key: "packaging-containers" + env: + PACKAGES: "{{matrix.ext}}" + PLATFORMS: "{{matrix.arch}}" + command: ".buildkite/scripts/steps/integration-package.sh" + artifact_paths: + - build/distributions/** + agents: + provider: "gcp" + machineType: "n2-standard-8" + matrix: + setup: + arch: + - linux/amd64 + - linux/arm64 + ext: + - rpm + - deb - label: "Serverless integration test" key: "serverless-integration-tests" depends_on: - - package-it + - int-packaging concurrency_group: elastic-agent-extended-testing/serverless-integration concurrency: 8 env: # we run each step in a different data center to spread the load TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a" command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' + buildkite-agent artifact download "build/distributions/**" . $BUILDKITE_BUILD_ID .buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite artifact_paths: - "build/TEST-**" - "build/diagnostics/*" agents: provider: "gcp" - machineType: "n1-standard-8" + machineType: "n2-standard-8" notify: - github_commit_status: context: "buildkite/elastic-agent-extended-testing - Serverless integration test" @@ -39,46 +89,47 @@ steps: - label: "Extended runtime leak tests" key: "extended-integration-tests" depends_on: - - package-it + - int-packaging concurrency_group: elastic-agent-extended-testing/leak-tests concurrency: 8 env: TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-b" command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' + buildkite-agent artifact download "build/distributions/**" . $BUILDKITE_BUILD_ID .buildkite/scripts/steps/integration_tests.sh stateful integration:TestForResourceLeaks artifact_paths: - "build/TEST-**" - "build/diagnostics/*" agents: provider: "gcp" - machineType: "n1-standard-8" + machineType: "n2-standard-8" notify: - github_commit_status: context: "buildkite/elastic-agent-extended-testing - Extended runtime leak tests" - label: "Triggering Integration tests" depends_on: - - package-it + - int-packaging trigger: "elastic-agent-extended-testing-bk" build: commit: "${BUILDKITE_COMMIT}" branch: "${BUILDKITE_BRANCH}" - label: "Serverless Beats Tests" + # To speedup the build process only packaging-ubuntu-x86-64 artifact dependency is required depends_on: - - package-it + - packaging-ubuntu-x86-64 key: "serverless-beats-integration-tests" concurrency_group: elastic-agent-extended-testing/beats-integration concurrency: 8 env: TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a" command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' + buildkite-agent artifact download "build/distributions/**" . $BUILDKITE_BUILD_ID .buildkite/scripts/steps/beats_tests.sh agents: provider: "gcp" - machineType: "n1-standard-8" + machineType: "n2-standard-8" notify: - github_commit_status: context: "buildkite/elastic-agent-extended-testing - Serverless Beats Tests" diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 9e10ef0cb6b..354e158a929 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -13,7 +13,7 @@ "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it|extended))|^/test\\W*(?:extended|)", "skip_ci_labels": [ "skip-ci" ], "skip_target_branches": [ ], - "skip_ci_on_only_changed": [ "^.ci/", "^.github/", "^updatecli-compose.yaml", "^changelog", "^docs/", "\\.md$", "^docker-compose.yml", "^.pre-commit-config.yaml", "skaffold.yaml", "^Dockerfile.skaffold", "^Dockerfile"], + "skip_ci_on_only_changed": [ "^.ci/", "^.github/", "^updatecli-compose.yaml", "^changelog", "^docs/", "\\.md$", "^docker-compose.yml", "^.mergify.yml", "^.pre-commit-config.yaml", "skaffold.yaml", "^Dockerfile.skaffold", "^Dockerfile"], "always_require_ci_on_changed": [ ] }, { diff --git a/.buildkite/scripts/steps/integration-package.sh b/.buildkite/scripts/steps/integration-package.sh index 0ba7323946b..3ba5bf3d7b3 100755 --- a/.buildkite/scripts/steps/integration-package.sh +++ b/.buildkite/scripts/steps/integration-package.sh @@ -2,5 +2,11 @@ set -euo pipefail source .buildkite/scripts/common.sh + # Remove AGENT_PACKAGE_VERSION pinning as soon as 9.0.0 is released -AGENT_PACKAGE_VERSION=9.0.0 PACKAGES=tar.gz,zip,rpm,deb PLATFORMS=linux/amd64,linux/arm64,windows/amd64 SNAPSHOT=true EXTERNAL=true mage package +export AGENT_PACKAGE_VERSION=9.0.0 + +export SNAPSHOT="true" +export EXTERNAL="true" + +mage package diff --git a/.mergify.yml b/.mergify.yml index 6d3baa92534..1ad95ec6f1b 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -374,6 +374,19 @@ pull_request_rules: labels: - "backport" title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}" + - name: backport patches to 9.0 branch + conditions: + - merged + - label=backport-9.0 + actions: + backport: + assignees: + - "{{ author }}" + branches: + - "9.0" + labels: + - "backport" + title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}" - name: backport patches to all active minor branches for the 8 major. conditions: @@ -386,6 +399,7 @@ pull_request_rules: # NOTE: this list needs to be changed when a new minor branch is created # or an existing minor branch reached EOL. branches: + - "8.x" - "8.18" - "8.17" - "8.16" diff --git a/changelog/fragments/1738874791-Fix-secret_paths-redaction-along-complex-paths.yaml b/changelog/fragments/1738874791-Fix-secret_paths-redaction-along-complex-paths.yaml new file mode 100644 index 00000000000..9346d6a1e5c --- /dev/null +++ b/changelog/fragments/1738874791-Fix-secret_paths-redaction-along-complex-paths.yaml @@ -0,0 +1,32 @@ +# Kind can be one of: +# - breaking-change: a change to previously-documented behavior +# - deprecation: functionality that is being removed in a later release +# - bug-fix: fixes a problem in a previous version +# - enhancement: extends functionality but does not break or fix existing behavior +# - feature: new functionality +# - known-issue: problems that we are aware of in a given version +# - security: impacts on the security of a product or a user’s deployment. +# - upgrade: important information for someone upgrading from a prior version +# - other: does not fit into any of the other categories +kind: bug-fix + +# Change summary; a 80ish characters long description of the change. +summary: Fix secret_paths redaction along complex paths + +# Long description; in case the summary is not enough to describe the change +# this field accommodate a description without length limits. +# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment. +#description: + +# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc. +component: elastic-agent + +# PR URL; optional; the PR number that added the changeset. +# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added. +# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number. +# Please provide it if you are adding a fragment for a different PR. +pr: https://github.com/elastic/elastic-agent/pull/6710 + +# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of). +# If not present is automatically filled by the tooling with the issue linked to the PR number. +#issue: https://github.com/owner/repo/1234 diff --git a/internal/pkg/diagnostics/diagnostics.go b/internal/pkg/diagnostics/diagnostics.go index 1e7ec30c7d6..f4b0d70ed90 100644 --- a/internal/pkg/diagnostics/diagnostics.go +++ b/internal/pkg/diagnostics/diagnostics.go @@ -398,7 +398,8 @@ func redactKey(k string) bool { strings.Contains(k, "passphrase") || strings.Contains(k, "password") || strings.Contains(k, "token") || - strings.Contains(k, "key") + strings.Contains(k, "key") || + strings.Contains(k, "secret") } func zipLogs(zw *zip.Writer, ts time.Time, topPath string, excludeEvents bool) error { @@ -593,7 +594,7 @@ func RedactSecretPaths(mapStr map[string]any, errOut io.Writer) map[string]any { fmt.Fprintln(errOut, "No output redaction: secret_paths attribute is not a list.") return mapStr } - cfg := ucfg.MustNewFrom(mapStr) + cfg := ucfg.MustNewFrom(mapStr, ucfg.PathSep(".")) for _, v := range arr { key, ok := v.(string) if !ok { @@ -601,11 +602,15 @@ func RedactSecretPaths(mapStr map[string]any, errOut io.Writer) map[string]any { continue } - if ok, _ := cfg.Has(key, -1, ucfg.PathSep(".")); ok { + if ok, err := cfg.Has(key, -1, ucfg.PathSep(".")); err != nil { + fmt.Fprintf(errOut, "Error redacting secret path %q: %v.\n", key, err) + } else if ok { err := cfg.SetString(key, -1, REDACTED, ucfg.PathSep(".")) if err != nil { fmt.Fprintf(errOut, "No output redaction for %q: %v.\n", key, err) } + } else { + fmt.Fprintf(errOut, "Unable to find secret path %q for redaction.\n", key) } } result, err := config.MustNewConfigFrom(cfg).ToMapStr() diff --git a/internal/pkg/diagnostics/diagnostics_test.go b/internal/pkg/diagnostics/diagnostics_test.go index 706ca550f24..87252cbf3e5 100644 --- a/internal/pkg/diagnostics/diagnostics_test.go +++ b/internal/pkg/diagnostics/diagnostics_test.go @@ -248,6 +248,43 @@ secret_paths: - inputs.0.redactKey - inputs.1.missingKey - outputs.default.redactOtherKey +`, + }, { + name: "path in nested list", + input: []byte(`id: test-policy +inputs: + - type: httpjson + data_stream: + namespace: default + streams: + - config_version: "2" + request.transforms: + - set: + target: header.Authorization + value: SSWS this-should-be-redacted + - set: + target: url.params.limit + value: "1000" +secret_paths: + - inputs.0.streams.0.request.transforms.0.set.value +`), + expect: `id: test-policy +inputs: + - data_stream: + namespace: default + streams: + - config_version: "2" + request: + transforms: + - set: + target: header.Authorization + value: + - set: + target: url.params.limit + value: "1000" + type: httpjson +secret_paths: + - inputs.0.streams.0.request.transforms.0.set.value `, }} @@ -255,9 +292,11 @@ secret_paths: t.Run(tc.name, func(t *testing.T) { file := client.DiagnosticFileResult{Content: tc.input, ContentType: "application/yaml"} var out bytes.Buffer - err := writeRedacted(io.Discard, &out, "testPath", file) + var errOut bytes.Buffer + err := writeRedacted(&errOut, &out, "testPath", file) require.NoError(t, err) + t.Logf("Error output: %s", errOut.String()) assert.Equal(t, tc.expect, out.String()) }) } diff --git a/testing/integration/upgrade_gpg_test.go b/testing/integration/upgrade_gpg_test.go index 9b7a01ac76f..8d1832b3650 100644 --- a/testing/integration/upgrade_gpg_test.go +++ b/testing/integration/upgrade_gpg_test.go @@ -8,6 +8,7 @@ package integration import ( "context" + "runtime" "strings" "testing" "time" @@ -98,6 +99,10 @@ func TestStandaloneUpgradeWithGPGFallbackOneRemoteFailing(t *testing.T) { Sudo: true, // requires Agent installation }) + if runtime.GOOS == "windows" { + t.Skip("This test is flaky on windows. See https://github.com/elastic/elastic-agent/issues/6732") + } + minVersion := upgradetest.Version_8_10_0_SNAPSHOT currentVersion, err := version.ParseVersion(define.Version()) require.NoError(t, err) diff --git a/testing/integration/upgrade_rollback_test.go b/testing/integration/upgrade_rollback_test.go index d84c4702873..1f35e0c7de4 100644 --- a/testing/integration/upgrade_rollback_test.go +++ b/testing/integration/upgrade_rollback_test.go @@ -168,6 +168,10 @@ func TestStandaloneUpgradeRollbackOnRestarts(t *testing.T) { Sudo: true, // requires Agent installation }) + if runtime.GOOS == "windows" { + t.Skip("This test is flaky on windows. See https://github.com/elastic/elastic-agent/issues/6733") + } + ctx, cancel := testcontext.WithDeadline(t, context.Background(), time.Now().Add(10*time.Minute)) defer cancel() diff --git a/testing/integration/upgrade_standalone_retry_test.go b/testing/integration/upgrade_standalone_retry_test.go index 7ee2e558cc8..6031d12a60b 100644 --- a/testing/integration/upgrade_standalone_retry_test.go +++ b/testing/integration/upgrade_standalone_retry_test.go @@ -13,6 +13,7 @@ import ( "net/http" "path" "path/filepath" + "runtime" "strings" "testing" "time" @@ -33,6 +34,10 @@ func TestStandaloneUpgradeRetryDownload(t *testing.T) { Sudo: true, // requires Agent installation }) + if runtime.GOOS == "windows" { + t.Skip("This test is flaky on windows. See https://github.com/elastic/elastic-agent/issues/6731") + } + ctx, cancel := testcontext.WithDeadline(t, context.Background(), time.Now().Add(10*time.Minute)) defer cancel() diff --git a/testing/integration/upgrade_standalone_same_commit_test.go b/testing/integration/upgrade_standalone_same_commit_test.go index 3d508f44a81..5759aa2fca6 100644 --- a/testing/integration/upgrade_standalone_same_commit_test.go +++ b/testing/integration/upgrade_standalone_same_commit_test.go @@ -18,6 +18,7 @@ import ( "os" "path" "path/filepath" + "runtime" "strings" "testing" "time" @@ -81,6 +82,11 @@ func TestStandaloneUpgradeSameCommit(t *testing.T) { }) t.Run(fmt.Sprintf("Upgrade on a repackaged version of agent %s (%s)", currentVersion, unPrivilegedString), func(t *testing.T) { + + if runtime.GOOS == "windows" { + t.Skip("This test is flaky on windows. See https://github.com/elastic/elastic-agent/issues/6729") + } + ctx, cancel := testcontext.WithDeadline(t, context.Background(), time.Now().Add(10*time.Minute)) defer cancel()