AMLII-2181 Add ssl scenario to jmxfetch test suite

.gitlab/e2e/e2e.yml

@@ -294,6 +294,7 @@ new-e2e-aml:
     - deploy_windows_testing-a7
     - qa_agent
     - qa_agent_jmx
+    - qa_agent_fips_jmx
     - qa_dca
   rules:
     - !reference [.on_aml_or_e2e_changes]

test/new-e2e/pkg/provisioners/aws/docker/host.go

@@ -19,6 +19,7 @@ import (
 	"github.com/DataDog/test-infra-definitions/components/datadog/apps/dogstatsd"
 	"github.com/DataDog/test-infra-definitions/components/datadog/dockeragentparams"
 	"github.com/DataDog/test-infra-definitions/components/docker"
+	"github.com/DataDog/test-infra-definitions/components/remote"
 	"github.com/DataDog/test-infra-definitions/resources/aws"
 	"github.com/DataDog/test-infra-definitions/scenarios/aws/ec2"
 	"github.com/DataDog/test-infra-definitions/scenarios/aws/fakeintake"
@@ -31,13 +32,16 @@ const (
 	defaultVMName     = "dockervm"
 )
 
+type setupFn func(*aws.Environment, *remote.Host, *ProvisionerParams) error
+
 // ProvisionerParams contains all the parameters needed to create the environment
 type ProvisionerParams struct {
 	name string
 
 	vmOptions         []ec2.VMOption
 	agentOptions      []dockeragentparams.Option
 	fakeintakeOptions []fakeintake.Option
+	setupCallbacks    []setupFn
 	extraConfigParams runner.ConfigMap
 	testingWorkload   bool
 }
@@ -130,6 +134,14 @@ func WithTestingWorkload() ProvisionerOption {
 	}
 }
 
+// WithSetupCallback adds a callback between host setup end and the agent starting up.
+func WithSetupCallback(cb setupFn) ProvisionerOption {
+	return func(params *ProvisionerParams) error {
+		params.setupCallbacks = append(params.setupCallbacks, cb)
+		return nil
+	}
+}
+
 // RunParams contains parameters for the run function
 type RunParams struct {
 	Environment       *aws.Environment
@@ -198,6 +210,12 @@ func Run(ctx *pulumi.Context, env *environments.DockerHost, runParams RunParams)
 		env.FakeIntake = nil
 	}
 
+	for _, cb := range params.setupCallbacks {
+		if err := cb(&awsEnv, host, params); err != nil {
+			return err
+		}
+	}
+
 	// Create Agent if required
 	if params.agentOptions != nil {
 		params.agentOptions = append(params.agentOptions, dockeragentparams.WithTags([]string{"stackid:" + ctx.Stack()}))

test/new-e2e/tests/agent-metrics-logs/jmxfetch/jmxfetch_nix_test.go

@@ -7,6 +7,9 @@ package jmxfetch
 
 import (
 	_ "embed"
+	"encoding/json"
+	"fmt"
+	"path"
 	"strings"
 	"testing"
 	"time"
@@ -17,47 +20,87 @@ import (
 	awsdocker "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/docker"
 	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient"
 
+	"github.com/DataDog/test-infra-definitions/common/config"
+	"github.com/DataDog/test-infra-definitions/common/utils"
+	"github.com/DataDog/test-infra-definitions/components/command"
 	"github.com/DataDog/test-infra-definitions/components/datadog/apps/jmxfetch"
 	"github.com/DataDog/test-infra-definitions/components/datadog/dockeragentparams"
 	"github.com/DataDog/test-infra-definitions/components/docker"
+	"github.com/DataDog/test-infra-definitions/components/remote"
+	"github.com/DataDog/test-infra-definitions/resources/aws"
 
+	"github.com/pulumi/pulumi-aws/sdk/v6/go/aws/secretsmanager"
 	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
-
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
 )
 
 //go:embed testdata/docker-labels.yaml
 var jmxFetchADLabels string
 
-var jmxFetchADLabelsDockerComposeManifest = docker.ComposeInlineManifest{
-	Name:    "jmx-test-app-labels",
-	Content: pulumi.String(jmxFetchADLabels),
-}
+const testCertsARN = "arn:aws:secretsmanager:us-east-1:376334461865:secret:agent-e2e-jmxfetch-test-certs-20250107-zDSge7"
 
 type jmxfetchNixTest struct {
 	e2e.BaseSuite[environments.DockerHost]
+
+	fips bool
 }
 
-func TestJMXFetchNix(t *testing.T) {
-	t.Parallel()
+func testJMXFetchNix(t *testing.T, mtls bool, fips bool) {
+	adLabelsManifest, err := makeADLabelsManifest(mtls, fips)
+	require.NoError(t, err)
+
+	extraManifests := []docker.ComposeInlineManifest{
+		jmxfetch.DockerComposeManifest,
+		*adLabelsManifest,
+	}
+
+	if mtls {
+		mtlsManifest, err := makeMtlsManifest(fips)
+		require.NoError(t, err)
+		extraManifests = append(extraManifests, *mtlsManifest)
+	}
+
+	// causes all sorts of `suite.go:493: unable to create session output directory:` errors
+	// due to race to create /home/vagrant/e2e-output/latest from every test
+	// t.Parallel()
+
 	suiteParams := []e2e.SuiteOption{e2e.WithProvisioner(
 		awsdocker.Provisioner(
 			awsdocker.WithAgentOptions(
 				dockeragentparams.WithLogs(),
 				dockeragentparams.WithJMX(),
-				dockeragentparams.WithExtraComposeInlineManifest(
-					jmxfetch.DockerComposeManifest,
-					jmxFetchADLabelsDockerComposeManifest,
-				),
-			)))}
+				choice(fips, dockeragentparams.WithFIPS(), none),
+				dockeragentparams.WithExtraComposeInlineManifest(extraManifests...),
+			),
+			choice(mtls, awsdocker.WithSetupCallback(fetchCertificates), none),
+		)),
+		e2e.WithStackName(fmt.Sprintf("jmxfetchnixtest-fips_%v-mtls_%v", fips, mtls)),
+	}
 
 	e2e.Run(t,
-		&jmxfetchNixTest{},
+		&jmxfetchNixTest{fips: fips},
 		suiteParams...,
 	)
 }
 
+func TestJMXFetchNix(t *testing.T) {
+	testJMXFetchNix(t, false, false)
+}
+
+func TestJMXFetchNixFIPS(t *testing.T) {
+	testJMXFetchNix(t, false, true)
+}
+
+func TestJMXFetchNixMtls(t *testing.T) {
+	testJMXFetchNix(t, true, false)
+}
+
+func TestJMXFetchNixMtlsFIPS(t *testing.T) {
+	testJMXFetchNix(t, true, true)
+}
+
 func (j *jmxfetchNixTest) Test_FakeIntakeReceivesJMXFetchMetrics() {
 	metricNames := []string{
 		"test.e2e.jmxfetch.counter_100",
@@ -85,7 +128,7 @@ func (j *jmxfetchNixTest) Test_FakeIntakeReceivesJMXFetchMetrics() {
 		for _, metricName := range metricNames {
 			tjc, err := j.Env().FakeIntake.Client().FilterMetrics(metricName)
 			assert.NoError(j.T(), err)
-			assert.NotEmpty(j.T(), tjc, "Filter metrics was empty for", metricName)
+			assert.NotEmpty(j.T(), tjc, "Filter metrics was empty for %q", metricName)
 			if len(tjc) > 0 {
 				for _, point := range tjc[0].Points {
 					j.T().Logf("Found metrics: %q \n%v - %v \n%q", tjc[0].Metric, point, point.Value, tjc[0].Type)
@@ -135,3 +178,202 @@ func (j *jmxfetchNixTest) TestJMXListCollectedWithRateMetrics() {
 		}
 	}
 }
+
+func (j *jmxfetchNixTest) TestJMXFIPSMode() {
+	env, err := j.Env().Docker.Client.ExecuteCommandWithErr(j.Env().Agent.ContainerName, "env")
+	require.NoError(j.T(), err)
+	if j.fips {
+		assert.Contains(j.T(), env, "JAVA_TOOL_OPTIONS=--module-path")
+	} else {
+		assert.Contains(j.T(), env, "JAVA_TOOL_OPTIONS=\n")
+	}
+}
+
+func none[T any](_ T) error { return nil }
+
+func choice[T any](cond bool, then, otherwise T) T {
+	if cond {
+		return then
+	}
+	return otherwise
+}
+
+func fetchCertificates(awsEnv *aws.Environment, h *remote.Host, params *awsdocker.ProvisionerParams) error {
+	args := secretsmanager.LookupSecretVersionOutputArgs{SecretId: pulumi.String(testCertsARN)}
+	certs := secretsmanager.LookupSecretVersionOutput(awsEnv.Ctx(), args, awsEnv.WithProvider(config.ProviderAWS))
+
+	// Commands here are queued and executed later.
+	mkdirCmd, path, err := h.OS.FileManager().HomeDirectory("certs")
+	if err != nil {
+		return fmt.Errorf("failed to create mkdir command: %w", err)
+	}
+	runner := h.OS.Runner()
+
+	unpack, err := runner.Command("certs", &command.Args{
+		Create: pulumi.String(fmt.Sprintf("base64 -d | tar -C%q -xzf-", path)),
+		Stdin:  certs.SecretBinary(),
+	}, utils.PulumiDependsOn(mkdirCmd))
+
+	if err != nil {
+		return fmt.Errorf("failed to create decode command: %w", err)
+	}
+
+	awsdocker.WithAgentOptions(dockeragentparams.WithPulumiDependsOn(utils.PulumiDependsOn(unpack)))(params)
+
+	return nil
+}
+
+type serviceDesc struct {
+	Secrets     []string
+	Labels      map[string]string
+	Environment map[string]string
+}
+
+type secretDesc struct {
+	File string
+}
+
+type composeDesc struct {
+	Services map[string]serviceDesc
+	Secrets  map[string]secretDesc
+}
+
+// makeMtlsManifest provides secrets and configures jmx test app to use ssl.
+func makeMtlsManifest(fips bool) (*docker.ComposeInlineManifest, error) {
+	appSecrets := []string{"pkcs12-java-app-keystore", "pkcs12-java-app-truststore"}
+	jmxfetchSecrets := []string{"pkcs12-jmxfetch-keystore", "pkcs12-jmxfetch-truststore"}
+	if fips {
+		jmxfetchSecrets = []string{"bcfks-jmxfetch-keystore", "bcfks-jmxfetch-truststore"}
+	}
+
+	appFlags := []string{
+		"-Dcom.sun.management.jmxremote.ssl.need.client.auth=true",
+		"-Djavax.net.ssl.keyStore=/run/secrets/pkcs12-java-app-keystore",
+		"-Djavax.net.ssl.keyStorePassword=changeit",
+		"-Djavax.net.ssl.trustStore=/run/secrets/pkcs12-java-app-truststore",
+		"-Djavax.net.ssl.trustStorePassword=changeit",
+	}
+
+	desc := composeDesc{
+		Services: map[string]serviceDesc{
+			"jmx-test-app": {
+				Secrets: appSecrets,
+				Environment: map[string]string{
+					"SSL_MODE":          "true",
+					"JAVA_TOOL_OPTIONS": strings.Join(appFlags, " "),
+					"DUMMY_UPDATE_2":    "remove_before_merging",
+				},
+			},
+			"agent": {
+				Secrets: jmxfetchSecrets,
+			},
+		},
+
+		Secrets: map[string]secretDesc{
+			"bcfks-java-app-keystore":    {"../certs/bcfks/java-app-keystore"},
+			"bcfks-java-app-truststore":  {"../certs/bcfks/java-app-truststore"},
+			"bcfks-jmxfetch-keystore":    {"../certs/bcfks/jmxfetch-keystore"},
+			"bcfks-jmxfetch-truststore":  {"../certs/bcfks/jmxfetch-truststore"},
+			"pkcs12-java-app-keystore":   {"../certs/pkcs12/java-app-keystore"},
+			"pkcs12-java-app-truststore": {"../certs/pkcs12/java-app-truststore"},
+			"pkcs12-jmxfetch-keystore":   {"../certs/pkcs12/jmxfetch-keystore"},
+			"pkcs12-jmxfetch-truststore": {"../certs/pkcs12/jmxfetch-truststore"},
+		},
+	}
+
+	out, err := yaml.Marshal(&desc)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal compose manifest yaml: %w", err)
+	}
+
+	return &docker.ComposeInlineManifest{
+		Name:    "jmx-test-tls-manifest",
+		Content: pulumi.String(string(out)),
+	}, nil
+}
+
+type checksLabel map[string]checkConfig
+
+type checkConfig struct {
+	InitConfig json.RawMessage `json:"init_config"`
+	Instances  []checkInstance
+}
+
+type checkInstance struct {
+	Host               string  `json:"host"`
+	Port               string  `json:"port"`
+	RmiRegistrySsl     *bool   `json:"rmi_registry_ssl,omitempty"`
+	KeyStorePath       *string `json:"key_store_path,omitempty"`
+	KeyStorePassword   *string `json:"key_store_password,omitempty"`
+	TrustStorePath     *string `json:"trust_store_path,omitempty"`
+	TrustStorePassword *string `json:"trust_store_password,omitempty"`
+}
+
+var defaultJavaPassword = "changeit"
+
+const adLabelName = "com.datadoghq.ad.checks"
+
+func makeADLabelsManifest(mtls bool, fips bool) (*docker.ComposeInlineManifest, error) {
+	manifestYaml := jmxFetchADLabels
+
+	if mtls {
+		var err error
+		manifestYaml, err = withInstance(manifestYaml, func(instance *checkInstance) {
+			secretPrefix := "/run/secrets"
+			keystore, truststore := "pkcs12-jmxfetch-keystore", "pkcs12-jmxfetch-truststore"
+			if fips {
+				keystore, truststore = "bcfks-jmxfetch-keystore", "bcfks-jmxfetch-truststore"
+			}
+			keystorePath := path.Join(secretPrefix, keystore)
+			truststorePath := path.Join(secretPrefix, truststore)
+
+			instance.RmiRegistrySsl = &mtls
+			instance.KeyStorePath = &keystorePath
+			instance.KeyStorePassword = &defaultJavaPassword
+			instance.TrustStorePath = &truststorePath
+			instance.TrustStorePassword = &defaultJavaPassword
+		})
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return &docker.ComposeInlineManifest{
+		Name:    "jmx-test-app-labels",
+		Content: pulumi.String(manifestYaml),
+	}, nil
+}
+
+// withInstance digs out check instance out of the testdata config and
+// puts it back after callback has modified it.
+func withInstance(manifestYaml string, cb func(*checkInstance)) (string, error) {
+	var manifest composeDesc
+	err := yaml.Unmarshal([]byte(manifestYaml), &manifest)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse label manifest: %w", err)
+	}
+
+	labels := manifest.Services["jmx-test-app"].Labels
+	labelJSON := []byte(labels[adLabelName])
+
+	var label checksLabel
+	err = json.Unmarshal(labelJSON, &label)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse label %q: %w", adLabelName, err)
+	}
+
+	cb(&label["test"].Instances[0])
+
+	labelJSON, err = json.Marshal(label)
+	if err != nil {
+		return "", fmt.Errorf("failed to marshal label json: %w", err)
+	}
+
+	labels[adLabelName] = string(labelJSON)
+	manifestBytes, err := yaml.Marshal(&manifest)
+	if err != nil {
+		return "", fmt.Errorf("failed to marshal manifest yaml: %w", err)
+	}
+
+	return string(manifestBytes), nil
+}