diff --git a/service/config/sample.yml b/service/config/sample.yml index 14486ddbf..7ed8b9373 100644 --- a/service/config/sample.yml +++ b/service/config/sample.yml @@ -215,7 +215,9 @@ svr2: -----END CERTIFICATE----- svr3: - uri: svr3.example.com + backend1Uri: backend1.example.com + backend2Uri: backend2.example.com + backend3Uri: backend3.example.com userAuthenticationTokenSharedSecret: secret://svr3.userAuthenticationTokenSharedSecret userIdTokenSharedSecret: secret://svr3.userIdTokenSharedSecret svrCaCertificates: diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java index 249c7d8ce..f10a22ec9 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java @@ -206,6 +206,7 @@ import org.whispersystems.textsecuregcm.s3.PostPolicyGenerator; import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient; import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery2Client; +import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery3Client; import org.whispersystems.textsecuregcm.spam.ChallengeConstraintChecker; import org.whispersystems.textsecuregcm.spam.RegistrationFraudChecker; import org.whispersystems.textsecuregcm.spam.RegistrationRecoveryChecker; @@ -478,8 +479,10 @@ public void run(WhisperServerConfiguration config, Environment environment) thro .maxThreads(1).minThreads(1).build(); ExecutorService fcmSenderExecutor = environment.lifecycle().executorService(name(getClass(), "fcmSender-%d")) .maxThreads(32).minThreads(32).workQueue(fcmSenderQueue).build(); - ExecutorService secureValueRecoveryServiceExecutor = environment.lifecycle() - .executorService(name(getClass(), "secureValueRecoveryService-%d")).maxThreads(1).minThreads(1).build(); + ExecutorService secureValueRecovery2ServiceExecutor = environment.lifecycle() + .executorService(name(getClass(), "secureValueRecoveryService2-%d")).maxThreads(1).minThreads(1).build(); + ExecutorService secureValueRecovery3ServiceExecutor = environment.lifecycle() + .executorService(name(getClass(), "secureValueRecoveryService3-%d")).maxThreads(1).minThreads(1).build(); ExecutorService storageServiceExecutor = environment.lifecycle() .executorService(name(getClass(), "storageService-%d")).maxThreads(1).minThreads(1).build(); ExecutorService virtualThreadEventLoggerExecutor = environment.lifecycle() @@ -602,7 +605,9 @@ public void run(WhisperServerConfiguration config, Environment environment) thro config.getKeyTransparencyServiceConfiguration().clientPrivateKey().value(), keyTransparencyCallbackExecutor); SecureValueRecovery2Client secureValueRecovery2Client = new SecureValueRecovery2Client(svr2CredentialsGenerator, - secureValueRecoveryServiceExecutor, secureValueRecoveryServiceRetryExecutor, config.getSvr2Configuration()); + secureValueRecovery2ServiceExecutor, secureValueRecoveryServiceRetryExecutor, config.getSvr2Configuration()); + SecureValueRecovery3Client secureValueRecovery3Client = new SecureValueRecovery3Client(svr3CredentialsGenerator, + secureValueRecovery3ServiceExecutor, secureValueRecoveryServiceRetryExecutor, config.getSvr3Configuration()); SecureStorageClient secureStorageClient = new SecureStorageClient(storageCredentialsGenerator, storageServiceExecutor, storageServiceRetryExecutor, config.getSecureStorageServiceConfiguration()); DisconnectionRequestManager disconnectionRequestManager = new DisconnectionRequestManager(pubsubClient, disconnectionRequestListenerExecutor); @@ -623,7 +628,7 @@ public void run(WhisperServerConfiguration config, Environment environment) thro new ClientPublicKeysManager(clientPublicKeys, accountLockManager, accountLockExecutor); AccountsManager accountsManager = new AccountsManager(accounts, phoneNumberIdentifiers, cacheCluster, pubsubClient, accountLockManager, keysManager, messagesManager, profilesManager, - secureStorageClient, secureValueRecovery2Client, disconnectionRequestManager, + secureStorageClient, secureValueRecovery2Client, secureValueRecovery3Client, disconnectionRequestManager, registrationRecoveryPasswordsManager, clientPublicKeysManager, accountLockExecutor, messagePollExecutor, clock, config.getLinkDeviceSecretConfiguration().secret().value(), dynamicConfigurationManager); RemoteConfigsManager remoteConfigsManager = new RemoteConfigsManager(remoteConfigs); diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/configuration/SecureValueRecovery3Configuration.java b/service/src/main/java/org/whispersystems/textsecuregcm/configuration/SecureValueRecovery3Configuration.java index 16f454405..4d46f125e 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/configuration/SecureValueRecovery3Configuration.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/configuration/SecureValueRecovery3Configuration.java @@ -13,7 +13,9 @@ import org.whispersystems.textsecuregcm.util.ExactlySize; public record SecureValueRecovery3Configuration( - @NotBlank String uri, + @NotBlank String backend1Uri, + @NotBlank String backend2Uri, + @NotBlank String backend3Uri, @ExactlySize(32) SecretBytes userAuthenticationTokenSharedSecret, @ExactlySize(32) SecretBytes userIdTokenSharedSecret, @NotEmpty List<@NotBlank String> svrCaCertificates, diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/securevaluerecovery/SecureValueRecovery3Client.java b/service/src/main/java/org/whispersystems/textsecuregcm/securevaluerecovery/SecureValueRecovery3Client.java new file mode 100644 index 000000000..33ae74c48 --- /dev/null +++ b/service/src/main/java/org/whispersystems/textsecuregcm/securevaluerecovery/SecureValueRecovery3Client.java @@ -0,0 +1,86 @@ +/* + * Copyright 2024 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.whispersystems.textsecuregcm.securevaluerecovery; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.net.HttpHeaders; +import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentials; +import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentialsGenerator; +import org.whispersystems.textsecuregcm.configuration.SecureValueRecovery3Configuration; +import org.whispersystems.textsecuregcm.http.FaultTolerantHttpClient; +import org.whispersystems.textsecuregcm.util.ExceptionUtils; +import org.whispersystems.textsecuregcm.util.HttpUtils; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.security.cert.CertificateException; +import java.time.Duration; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; +import java.util.concurrent.ScheduledExecutorService; +import java.util.stream.Stream; + +import static org.whispersystems.textsecuregcm.util.HeaderUtils.basicAuthHeader; + +/** + * A client for sending requests to Signal's secure value recovery v3 service on behalf of authenticated users. + */ +public class SecureValueRecovery3Client { + private final ExternalServiceCredentialsGenerator secureValueRecoveryCredentialsGenerator; + private final URI backend1Uri; + private final URI backend2Uri; + private final URI backend3Uri; + + private final FaultTolerantHttpClient httpClient; + + @VisibleForTesting + static final String DELETE_PATH = "/v1/delete"; + + public SecureValueRecovery3Client(final ExternalServiceCredentialsGenerator secureValueRecoveryCredentialsGenerator, + final Executor executor, final ScheduledExecutorService retryExecutor, + final SecureValueRecovery3Configuration configuration) + throws CertificateException { + this.secureValueRecoveryCredentialsGenerator = secureValueRecoveryCredentialsGenerator; + this.backend1Uri = URI.create(configuration.backend1Uri()).resolve(DELETE_PATH); + this.backend2Uri = URI.create(configuration.backend2Uri()).resolve(DELETE_PATH); + this.backend3Uri = URI.create(configuration.backend3Uri()).resolve(DELETE_PATH); + this.httpClient = FaultTolerantHttpClient.newBuilder() + .withCircuitBreaker(configuration.circuitBreaker()) + .withRetry(configuration.retry()) + .withRetryExecutor(retryExecutor) + .withVersion(HttpClient.Version.HTTP_1_1) + .withConnectTimeout(Duration.ofSeconds(10)) + .withRedirect(HttpClient.Redirect.NEVER) + .withExecutor(executor) + .withName("secure-value-recovery3") + .withSecurityProtocol(FaultTolerantHttpClient.SECURITY_PROTOCOL_TLS_1_2) + .withTrustedServerCertificates(configuration.svrCaCertificates().toArray(String[]::new)) + .build(); + } + + public CompletableFuture deleteBackups(final UUID accountUuid) { + final ExternalServiceCredentials credentials = secureValueRecoveryCredentialsGenerator.generateForUuid(accountUuid); + final List>> futures = Stream.of(backend1Uri, backend2Uri, backend3Uri) + .map(uri -> HttpRequest.newBuilder() + .uri(uri) + .DELETE() + .header(HttpHeaders.AUTHORIZATION, basicAuthHeader(credentials)) + .build()) + .map(request -> httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString())) + .toList(); + + return CompletableFuture.allOf(futures.toArray(CompletableFuture[]::new)) + .thenApply(ignored -> futures.stream().map(CompletableFuture::join).toList()) + .thenAccept(responses -> responses.forEach(response -> { + if (!HttpUtils.isSuccessfulResponse(response.statusCode())) { + throw new SecureValueRecoveryException(String.format("Failed to delete backup in %s", response.uri()), String.valueOf(response.statusCode())); + } + })); + } +} diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountsManager.java b/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountsManager.java index b3aea2470..5044fe80b 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountsManager.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/storage/AccountsManager.java @@ -82,6 +82,7 @@ import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClusterClient; import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient; import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery2Client; +import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery3Client; import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecoveryException; import org.whispersystems.textsecuregcm.util.DestinationDeviceValidator; import org.whispersystems.textsecuregcm.util.ExceptionUtils; @@ -125,6 +126,8 @@ public class AccountsManager extends RedisPubSubAdapter implemen private final ProfilesManager profilesManager; private final SecureStorageClient secureStorageClient; private final SecureValueRecovery2Client secureValueRecovery2Client; + private final SecureValueRecovery3Client secureValueRecovery3Client; + private final DisconnectionRequestManager disconnectionRequestManager; private final RegistrationRecoveryPasswordsManager registrationRecoveryPasswordsManager; private final ClientPublicKeysManager clientPublicKeysManager; @@ -207,6 +210,7 @@ public AccountsManager(final Accounts accounts, final ProfilesManager profilesManager, final SecureStorageClient secureStorageClient, final SecureValueRecovery2Client secureValueRecovery2Client, + final SecureValueRecovery3Client secureValueRecovery3Client, final DisconnectionRequestManager disconnectionRequestManager, final RegistrationRecoveryPasswordsManager registrationRecoveryPasswordsManager, final ClientPublicKeysManager clientPublicKeysManager, @@ -225,6 +229,7 @@ public AccountsManager(final Accounts accounts, this.profilesManager = profilesManager; this.secureStorageClient = secureStorageClient; this.secureValueRecovery2Client = secureValueRecovery2Client; + this.secureValueRecovery3Client = secureValueRecovery3Client; this.disconnectionRequestManager = disconnectionRequestManager; this.registrationRecoveryPasswordsManager = requireNonNull(registrationRecoveryPasswordsManager); this.clientPublicKeysManager = clientPublicKeysManager; @@ -1248,19 +1253,27 @@ private CompletableFuture delete(final Account account) { account.getIdentifier(IdentityType.ACI), device.getId()))) .toList(); - CompletableFuture deleteBackupFuture = secureValueRecovery2Client.deleteBackups(account.getUuid()) + final CompletableFuture svr2DeleteBackupFuture = secureValueRecovery2Client.deleteBackups(account.getUuid()) .exceptionally(ExceptionUtils.exceptionallyHandler(SecureValueRecoveryException.class, exception -> { final List svrStatusCodesToIgnore = dynamicConfigurationManager.getConfiguration().getSvrStatusCodesToIgnoreForAccountDeletion(); if (svrStatusCodesToIgnore.contains(exception.getStatusCode())) { - logger.warn("Failed to delete backup for account: " + account.getUuid(), exception); + logger.warn("Ignoring failure to delete svr2 backup for account: " + account.getUuid(), exception); return null; } + logger.warn("Failed to delete svr2 backup for account: " + account.getUuid(), exception); throw new CompletionException(exception); })); + final CompletableFuture svr3DeleteBackupFuture = secureValueRecovery3Client.deleteBackups(account.getUuid()) + .exceptionally(exception -> { + // We don't care about errors from SVR3 because we're not currently using it + return null; + }); + return CompletableFuture.allOf( secureStorageClient.deleteStoredData(account.getUuid()), - deleteBackupFuture, + svr2DeleteBackupFuture, + svr3DeleteBackupFuture, keysManager.deleteSingleUsePreKeys(account.getUuid()), keysManager.deleteSingleUsePreKeys(account.getPhoneNumberIdentifier()), messagesManager.clear(account.getUuid()), diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/workers/CommandDependencies.java b/service/src/main/java/org/whispersystems/textsecuregcm/workers/CommandDependencies.java index a05839ce3..18122b1f3 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/workers/CommandDependencies.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/workers/CommandDependencies.java @@ -32,6 +32,7 @@ import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration; import org.whispersystems.textsecuregcm.controllers.SecureStorageController; import org.whispersystems.textsecuregcm.controllers.SecureValueRecovery2Controller; +import org.whispersystems.textsecuregcm.controllers.SecureValueRecovery3Controller; import org.whispersystems.textsecuregcm.experiment.PushNotificationExperimentSamples; import org.whispersystems.textsecuregcm.limits.RateLimiters; import org.whispersystems.textsecuregcm.metrics.MicrometerAwsSdkMetricPublisher; @@ -44,6 +45,7 @@ import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClusterClient; import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient; import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery2Client; +import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery3Client; import org.whispersystems.textsecuregcm.storage.AccountLockManager; import org.whispersystems.textsecuregcm.storage.Accounts; import org.whispersystems.textsecuregcm.storage.AccountsManager; @@ -153,8 +155,10 @@ static CommandDependencies build( ExternalServiceCredentialsGenerator storageCredentialsGenerator = SecureStorageController.credentialsGenerator( configuration.getSecureStorageServiceConfiguration()); - ExternalServiceCredentialsGenerator secureValueRecoveryCredentialsGenerator = SecureValueRecovery2Controller.credentialsGenerator( + ExternalServiceCredentialsGenerator secureValueRecovery2CredentialsGenerator = SecureValueRecovery2Controller.credentialsGenerator( configuration.getSvr2Configuration()); + ExternalServiceCredentialsGenerator secureValueRecovery3CredentialsGenerator = SecureValueRecovery3Controller.credentialsGenerator( + configuration.getSvr3Configuration()); final ExecutorService awsSdkMetricsExecutor = environment.lifecycle() .virtualExecutorService(MetricRegistry.name(CommandDependencies.class, "awsSdkMetrics-%d")); @@ -204,9 +208,13 @@ static CommandDependencies build( FaultTolerantRedisClusterClient rateLimitersCluster = configuration.getRateLimitersCluster().build("rate_limiters", redisClientResourcesBuilder); SecureValueRecovery2Client secureValueRecovery2Client = new SecureValueRecovery2Client( - secureValueRecoveryCredentialsGenerator, secureValueRecoveryServiceExecutor, + secureValueRecovery2CredentialsGenerator, secureValueRecoveryServiceExecutor, secureValueRecoveryServiceRetryExecutor, configuration.getSvr2Configuration()); + SecureValueRecovery3Client secureValueRecovery3Client = new SecureValueRecovery3Client( + secureValueRecovery3CredentialsGenerator, secureValueRecoveryServiceExecutor, + secureValueRecoveryServiceRetryExecutor, + configuration.getSvr3Configuration()); SecureStorageClient secureStorageClient = new SecureStorageClient(storageCredentialsGenerator, storageServiceExecutor, storageServiceRetryExecutor, configuration.getSecureStorageServiceConfiguration()); DisconnectionRequestManager disconnectionRequestManager = new DisconnectionRequestManager(pubsubClient, disconnectionRequestListenerExecutor); @@ -228,7 +236,7 @@ static CommandDependencies build( new RegistrationRecoveryPasswordsManager(registrationRecoveryPasswords); AccountsManager accountsManager = new AccountsManager(accounts, phoneNumberIdentifiers, cacheCluster, pubsubClient, accountLockManager, keys, messagesManager, profilesManager, - secureStorageClient, secureValueRecovery2Client, disconnectionRequestManager, + secureStorageClient, secureValueRecovery2Client, secureValueRecovery3Client, disconnectionRequestManager, registrationRecoveryPasswordsManager, clientPublicKeysManager, accountLockExecutor, messagePollExecutor, clock, configuration.getLinkDeviceSecretConfiguration().secret().value(), dynamicConfigurationManager); RateLimiters rateLimiters = RateLimiters.createAndValidate(configuration.getLimitsConfiguration(), diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/controllers/SecureValueRecovery3ControllerTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/controllers/SecureValueRecovery3ControllerTest.java index d7d1f79e1..6783cd9c2 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/controllers/SecureValueRecovery3ControllerTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/controllers/SecureValueRecovery3ControllerTest.java @@ -55,6 +55,8 @@ public class SecureValueRecovery3ControllerTest extends SecureValueRecoveryControllerBaseTest { private static final SecureValueRecovery3Configuration CFG = new SecureValueRecovery3Configuration( + "", + "", "", randomSecretBytes(32), randomSecretBytes(32), diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/securevaluerecovery/SecureValueRecovery3ClientTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/securevaluerecovery/SecureValueRecovery3ClientTest.java new file mode 100644 index 000000000..19b7dde07 --- /dev/null +++ b/service/src/test/java/org/whispersystems/textsecuregcm/securevaluerecovery/SecureValueRecovery3ClientTest.java @@ -0,0 +1,196 @@ +/* + * Copyright 2024 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.whispersystems.textsecuregcm.securevaluerecovery; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.delete; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.whispersystems.textsecuregcm.util.MockUtils.randomSecretBytes; + +import com.github.tomakehurst.wiremock.junit5.WireMockExtension; +import java.security.cert.CertificateException; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentials; +import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentialsGenerator; +import org.whispersystems.textsecuregcm.configuration.SecureValueRecovery3Configuration; + +class SecureValueRecovery3ClientTest { + + private UUID accountUuid; + private ExternalServiceCredentialsGenerator credentialsGenerator; + private ExecutorService httpExecutor; + private ScheduledExecutorService retryExecutor; + + private SecureValueRecovery3Client secureValueRecovery3Client; + + @RegisterExtension + private static final WireMockExtension backend1WireMock = WireMockExtension.newInstance() + .options(wireMockConfig().dynamicPort().dynamicHttpsPort()) + .build(); + + @RegisterExtension + private static final WireMockExtension backend2WireMock = WireMockExtension.newInstance() + .options(wireMockConfig().dynamicPort().dynamicHttpsPort()) + .build(); + + @RegisterExtension + private static final WireMockExtension backend3WireMock = WireMockExtension.newInstance() + .options(wireMockConfig().dynamicPort().dynamicHttpsPort()) + .build(); + + @BeforeEach + void setUp() throws CertificateException { + accountUuid = UUID.randomUUID(); + credentialsGenerator = mock(ExternalServiceCredentialsGenerator.class); + httpExecutor = Executors.newSingleThreadExecutor(); + retryExecutor = Executors.newSingleThreadScheduledExecutor(); + + final SecureValueRecovery3Configuration config = new SecureValueRecovery3Configuration( + "http://localhost:" + backend1WireMock.getPort(), + "http://localhost:" + backend2WireMock.getPort(), + "http://localhost:" + backend3WireMock.getPort(), + randomSecretBytes(32), + randomSecretBytes(32), + // This is a randomly-generated, throwaway certificate that's not actually connected to anything + List.of(""" + -----BEGIN CERTIFICATE----- + MIICZDCCAc2gAwIBAgIBADANBgkqhkiG9w0BAQ0FADBPMQswCQYDVQQGEwJ1czEL + MAkGA1UECAwCVVMxHjAcBgNVBAoMFVNpZ25hbCBNZXNzZW5nZXIsIExMQzETMBEG + A1UEAwwKc2lnbmFsLm9yZzAeFw0yMDEyMjMyMjQ3NTlaFw0zMDEyMjEyMjQ3NTla + ME8xCzAJBgNVBAYTAnVzMQswCQYDVQQIDAJVUzEeMBwGA1UECgwVU2lnbmFsIE1l + c3NlbmdlciwgTExDMRMwEQYDVQQDDApzaWduYWwub3JnMIGfMA0GCSqGSIb3DQEB + AQUAA4GNADCBiQKBgQCfSLcZNHYqbxSsgWp4JvbPRHjQTrlsrKrgD2q7f/OY6O3Y + /X0QNcNSOJpliN8rmzwslfsrXHO3q1diGRw4xHogUJZ/7NQrHiP/zhN0VTDh49pD + ZpjXVyUbayLS/6qM5arKxBspzEFBb5v8cF6bPr76SO/rpGXiI0j6yJKX6fRiKwID + AQABo1AwTjAdBgNVHQ4EFgQU6Jrs/Fmj0z4dA3wvdq/WqA4P49IwHwYDVR0jBBgw + FoAU6Jrs/Fmj0z4dA3wvdq/WqA4P49IwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0B + AQ0FAAOBgQB+5d5+NtzLILfrc9QmJdIO1YeDP64JmFwTER0kEUouRsb9UwknVWZa + y7MTM4NoBV1k0zb5LAk89SIDPr/maW5AsLtEomzjnEiomjoMBUdNe3YCgQReoLnr + R/QaUNbrCjTGYfBsjGbIzmkWPUyTec2ZdRyJ8JiVl386+6CZkxnndQ== + -----END CERTIFICATE----- + """, """ + -----BEGIN CERTIFICATE----- + MIIEpDCCAowCCQC43PUTWSADVjANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDDAls + b2NhbGhvc3QwHhcNMjIxMDE3MjA0NTM0WhcNMjMxMDE3MjA0NTM0WjAUMRIwEAYD + VQQDDAlsb2NhbGhvc3QwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDV + x1cdEd2ffQTlTXWRiCHGcrlYf4RJnctt9sw/BuHWTLXBu5LhyJSGn5LRszO/NCXK + Z/cmGR7pLj366RtiwL+Qo3nhvDCK7T9xZeNIusM6XMcMK9D/DGCYPqtjQz8NXd9V + ajBBe6nwTDTa+oqX8Mt89foWNkg5Il/lY62u9Dr18LRZ2W9zzYi3Q9/K0CbIX6pM + yVlPIO5rITOR2IsbeyqsO9jufgX5lP4ZKLLBAP1b7usjC4YdvWacjQg/rK5aay1x + jC2HCDgo/4N30QVXzSA9nFfSe6AE/xkStK4819JqOkY5JsJCbef1P3hOOdSLEjbp + xq3MjOs6G6dOgteaAGs10vx7dHxDWETTIiD7BIZ9zRYgOF5bkCaIUO+JfySE1MHD + KBAFLoRuvmRev5Ln5R0MCHpUMSmMNgJqz+RWZV3g/gpYbuWiHgJOwL1393eK50Bg + W7SXQ8EjJj2yXZSH+1gPzN0DRoJZiaBoTPnCL2qUgvwFpW1PJsM5FDyUJFUoK5kK + HLBBSKAPt6ZlSrUe2nBgJv7EF1GK+fTU08LXgW33OpLceGPa0zTShkukQUMtUtZ8 + GqhO12ohMzEupIu5Xurthq4VVUrzHUdj1ZZRMhAbfLU36sd03MMyL/xBqTN6dzCa + GDGIPGpYjAllZ5xMRt2kZdv+Kr6oo3u2nLUIsqI7KQIDAQABMA0GCSqGSIb3DQEB + CwUAA4ICAQCB5s43YF35ssf5YONW5iAaifGpi1o0866xfeOybtohFGvQ7V2W34i9 + TYBCt8+0hgatMcvZ08f0vqig1i7nrvYcE1hnhL7JNkU8qm0s9ytHZt6j62nB0kd/ + uqE2hOEQalTf/2TGPV0CCgiqLyd8lEUQvQeA38wktwUeZpVnErlzHeMR2CvV3K8R + u4vV6SnBcf+TAt56RKYZkPyvZj5llQPo14Glyoo8qZES7Ky1SHmM0GL+baPRBjRW + 3KgSt98Wyu4yr9qu21JpnbAnLhBfzfSKjSeCRgFElUE1GIaFGRZ7ypA74dUKeLnb + /VUWrszmUhGaEjV9dpI6x6B/kSpQMtIQqBaKRY2ALUeEujS/rURi4iMDwSU+GkSH + cyEvZKS97OA/dWeXfLXdo4beDBRG93bI4rQnDg5+VdlBOkQSLueb8x6/VThMoC5d + vZiotFQHseljQAdTkNa6tBu6c4XDYPCKB3CfkMYOlCfTS7Acn5G6dxTPKBtLGBnL + nQfYyzuwYkN09+2PVzt6auBHr3To7uoclkxX+hxyvPIwIZ0N6b4tQR1FCAkvg29Q + WIOjZOKGW690ESKCKOnFjUHVO0HpuWnT81URTuY62FXsYdVc2wE4v0E04mEbqQ0P + lY6ZKNA81Lm3YADYtObmK1IUrOPo9BeIaPy0UM08SmN880Vunqa91Q== + -----END CERTIFICATE----- + """), + null, null); + + secureValueRecovery3Client = new SecureValueRecovery3Client(credentialsGenerator, httpExecutor, retryExecutor, + config); + } + + @AfterEach + void tearDown() throws InterruptedException { + httpExecutor.shutdown(); + httpExecutor.awaitTermination(1, TimeUnit.SECONDS); + retryExecutor.shutdown(); + retryExecutor.awaitTermination(1, TimeUnit.SECONDS); + } + + @Test + void deleteStoredData() { + final String username = RandomStringUtils.secure().nextAlphabetic(16); + final String password = RandomStringUtils.secure().nextAlphanumeric(32); + + when(credentialsGenerator.generateForUuid(accountUuid)).thenReturn( + new ExternalServiceCredentials(username, password)); + + backend1WireMock.stubFor(delete(urlEqualTo(SecureValueRecovery3Client.DELETE_PATH)) + .withBasicAuth(username, password) + .willReturn(aResponse().withStatus(202))); + + backend2WireMock.stubFor(delete(urlEqualTo(SecureValueRecovery3Client.DELETE_PATH)) + .withBasicAuth(username, password) + .willReturn(aResponse().withStatus(202))); + + backend3WireMock.stubFor(delete(urlEqualTo(SecureValueRecovery3Client.DELETE_PATH)) + .withBasicAuth(username, password) + .willReturn(aResponse().withStatus(202))); + + assertDoesNotThrow(() -> secureValueRecovery3Client.deleteBackups(accountUuid).join()); + } + + @ParameterizedTest + @MethodSource + void deleteStoredDataFailure(final int backend1Status, final int backend2Status, final int backend3Status) { + final String username = RandomStringUtils.secure().nextAlphabetic(16); + final String password = RandomStringUtils.secure().nextAlphanumeric(32); + + when(credentialsGenerator.generateForUuid(accountUuid)).thenReturn( + new ExternalServiceCredentials(username, password)); + + backend1WireMock.stubFor(delete(urlEqualTo(SecureValueRecovery3Client.DELETE_PATH)) + .withBasicAuth(username, password) + .willReturn(aResponse().withStatus(backend1Status))); + + backend2WireMock.stubFor(delete(urlEqualTo(SecureValueRecovery3Client.DELETE_PATH)) + .withBasicAuth(username, password) + .willReturn(aResponse().withStatus(backend2Status))); + + backend3WireMock.stubFor(delete(urlEqualTo(SecureValueRecovery3Client.DELETE_PATH)) + .withBasicAuth(username, password) + .willReturn(aResponse().withStatus(backend3Status))); + + final CompletionException completionException = assertThrows(CompletionException.class, + () -> secureValueRecovery3Client.deleteBackups(accountUuid).join()); + + assertInstanceOf(SecureValueRecoveryException.class, completionException.getCause()); + } + + private static Stream deleteStoredDataFailure() { + return Stream.of( + Arguments.of(400, 202, 202), + Arguments.of(202, 400, 202), + Arguments.of(202, 202, 400) + ); + } + +} diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountCreationDeletionIntegrationTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountCreationDeletionIntegrationTest.java index 57ca4652a..74de894a0 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountCreationDeletionIntegrationTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountCreationDeletionIntegrationTest.java @@ -49,6 +49,7 @@ import org.whispersystems.textsecuregcm.redis.RedisClusterExtension; import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient; import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery2Client; +import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery3Client; import org.whispersystems.textsecuregcm.tests.util.KeysHelper; public class AccountCreationDeletionIntegrationTest { @@ -125,6 +126,9 @@ void setUp() { final SecureValueRecovery2Client svr2Client = mock(SecureValueRecovery2Client.class); when(svr2Client.deleteBackups(any())).thenReturn(CompletableFuture.completedFuture(null)); + final SecureValueRecovery3Client svr3Client = mock(SecureValueRecovery3Client.class); + when(svr3Client.deleteBackups(any())).thenReturn(CompletableFuture.completedFuture(null)); + final PhoneNumberIdentifiers phoneNumberIdentifiers = new PhoneNumberIdentifiers(DYNAMO_DB_EXTENSION.getDynamoDbAsyncClient(), DynamoDbExtensionSchema.Tables.PNI.tableName()); @@ -155,6 +159,7 @@ void setUp() { profilesManager, secureStorageClient, svr2Client, + svr3Client, disconnectionRequestManager, registrationRecoveryPasswordsManager, clientPublicKeysManager, diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerChangeNumberIntegrationTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerChangeNumberIntegrationTest.java index 20206f18c..dd8555631 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerChangeNumberIntegrationTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerChangeNumberIntegrationTest.java @@ -41,6 +41,7 @@ import org.whispersystems.textsecuregcm.redis.RedisClusterExtension; import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient; import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery2Client; +import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery3Client; import org.whispersystems.textsecuregcm.storage.DynamoDbExtensionSchema.Tables; import org.whispersystems.textsecuregcm.tests.util.AccountsHelper; import org.whispersystems.textsecuregcm.tests.util.KeysHelper; @@ -117,6 +118,9 @@ void setup() throws InterruptedException { final SecureValueRecovery2Client svr2Client = mock(SecureValueRecovery2Client.class); when(svr2Client.deleteBackups(any())).thenReturn(CompletableFuture.completedFuture(null)); + final SecureValueRecovery3Client svr3Client = mock(SecureValueRecovery3Client.class); + when(svr3Client.deleteBackups(any())).thenReturn(CompletableFuture.completedFuture(null)); + disconnectionRequestManager = mock(DisconnectionRequestManager.class); final PhoneNumberIdentifiers phoneNumberIdentifiers = @@ -145,6 +149,7 @@ void setup() throws InterruptedException { profilesManager, secureStorageClient, svr2Client, + svr3Client, disconnectionRequestManager, registrationRecoveryPasswordsManager, clientPublicKeysManager, diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerConcurrentModificationIntegrationTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerConcurrentModificationIntegrationTest.java index b363cdd38..92127c526 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerConcurrentModificationIntegrationTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerConcurrentModificationIntegrationTest.java @@ -54,6 +54,7 @@ import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClient; import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient; import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery2Client; +import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery3Client; import org.whispersystems.textsecuregcm.storage.DynamoDbExtensionSchema.Tables; import org.whispersystems.textsecuregcm.tests.util.DevicesHelper; import org.whispersystems.textsecuregcm.tests.util.JsonHelpers; @@ -136,6 +137,7 @@ void setup() throws InterruptedException { mock(ProfilesManager.class), mock(SecureStorageClient.class), mock(SecureValueRecovery2Client.class), + mock(SecureValueRecovery3Client.class), mock(DisconnectionRequestManager.class), mock(RegistrationRecoveryPasswordsManager.class), mock(ClientPublicKeysManager.class), diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerDeviceTransferIntegrationTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerDeviceTransferIntegrationTest.java index c1f6da571..0f2780b14 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerDeviceTransferIntegrationTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerDeviceTransferIntegrationTest.java @@ -21,6 +21,7 @@ import org.whispersystems.textsecuregcm.redis.RedisServerExtension; import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient; import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery2Client; +import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery3Client; import java.nio.charset.StandardCharsets; import java.time.Clock; import java.time.Duration; @@ -65,6 +66,7 @@ void setUp() { mock(ProfilesManager.class), mock(SecureStorageClient.class), mock(SecureValueRecovery2Client.class), + mock(SecureValueRecovery3Client.class), mock(DisconnectionRequestManager.class), mock(RegistrationRecoveryPasswordsManager.class), mock(ClientPublicKeysManager.class), diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerTest.java index ce9338a28..b1077c168 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerTest.java @@ -87,6 +87,7 @@ import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClusterClient; import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient; import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery2Client; +import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery3Client; import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecoveryException; import org.whispersystems.textsecuregcm.storage.AccountsManager.UsernameReservation; import org.whispersystems.textsecuregcm.tests.util.AccountsHelper; @@ -130,6 +131,7 @@ class AccountsManagerTest { private RedisAdvancedClusterAsyncCommands asyncClusterCommands; private AccountsManager accountsManager; private SecureValueRecovery2Client svr2Client; + private SecureValueRecovery3Client svr3Client; private DynamicConfiguration dynamicConfiguration; private static final Answer ACCOUNT_UPDATE_ANSWER = (answer) -> { @@ -193,6 +195,9 @@ void setup() throws InterruptedException { svr2Client = mock(SecureValueRecovery2Client.class); when(svr2Client.deleteBackups(any())).thenReturn(CompletableFuture.completedFuture(null)); + svr3Client = mock(SecureValueRecovery3Client.class); + when(svr3Client.deleteBackups(any())).thenReturn(CompletableFuture.completedFuture(null)); + final PhoneNumberIdentifiers phoneNumberIdentifiers = mock(PhoneNumberIdentifiers.class); phoneNumberIdentifiersByE164 = new HashMap<>(); @@ -254,6 +259,7 @@ void setup() throws InterruptedException { profilesManager, storageClient, svr2Client, + svr3Client, disconnectionRequestManager, registrationRecoveryPasswordsManager, clientPublicKeysManager, @@ -266,7 +272,7 @@ void setup() throws InterruptedException { @ParameterizedTest @MethodSource - void testDeleteWithSvrErrorStatusCodes(final String statusCode, final boolean expectError) throws InterruptedException { + void testDeleteWithSvr2ErrorStatusCodes(final String statusCode, final boolean expectError) throws InterruptedException { when(svr2Client.deleteBackups(any())).thenReturn( CompletableFuture.failedFuture(new SecureValueRecoveryException("Failed to delete backup", statusCode))); when(dynamicConfiguration.getSvrStatusCodesToIgnoreForAccountDeletion()).thenReturn(List.of("500")); @@ -282,13 +288,26 @@ void testDeleteWithSvrErrorStatusCodes(final String statusCode, final boolean ex } } - private static Stream testDeleteWithSvrErrorStatusCodes() { + private static Stream testDeleteWithSvr2ErrorStatusCodes() { return Stream.of( Arguments.of("500", false), Arguments.of("429", true) ); } + @ParameterizedTest + @ValueSource(strings = {"500", "429"}) + void testDeleteWithSvr3ErrorStatusCodes(final String statusCode) throws InterruptedException { + when(svr3Client.deleteBackups(any())).thenReturn( + CompletableFuture.failedFuture(new SecureValueRecoveryException("Failed to delete backup", statusCode))); + + final AccountAttributes attributes = new AccountAttributes(false, 1, 2, null, null, true, null); + + final Account createdAccount = createAccount("+18005550123", attributes); + + assertDoesNotThrow(() -> accountsManager.delete(createdAccount, AccountsManager.DeletionReason.USER_REQUEST).toCompletableFuture().join()); + } + @Test void testGetByServiceIdentifier() { final UUID aci = UUID.randomUUID(); diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerUsernameIntegrationTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerUsernameIntegrationTest.java index 523015337..e145b2f9b 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerUsernameIntegrationTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/storage/AccountsManagerUsernameIntegrationTest.java @@ -41,6 +41,7 @@ import org.whispersystems.textsecuregcm.redis.RedisClusterExtension; import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient; import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery2Client; +import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery3Client; import org.whispersystems.textsecuregcm.storage.DynamoDbExtensionSchema.Tables; import org.whispersystems.textsecuregcm.tests.util.AccountsHelper; import org.whispersystems.textsecuregcm.util.AttributeValues; @@ -150,6 +151,7 @@ private void buildAccountsManager(final int initialWidth, int discriminatorMaxWi profileManager, mock(SecureStorageClient.class), mock(SecureValueRecovery2Client.class), + mock(SecureValueRecovery3Client.class), disconnectionRequestManager, mock(RegistrationRecoveryPasswordsManager.class), mock(ClientPublicKeysManager.class), diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AddRemoveDeviceIntegrationTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/storage/AddRemoveDeviceIntegrationTest.java index cf64e9904..9e7bd7292 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/storage/AddRemoveDeviceIntegrationTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/storage/AddRemoveDeviceIntegrationTest.java @@ -41,6 +41,7 @@ import org.whispersystems.textsecuregcm.redis.RedisServerExtension; import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient; import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery2Client; +import org.whispersystems.textsecuregcm.securevaluerecovery.SecureValueRecovery3Client; import org.whispersystems.textsecuregcm.tests.util.AccountsHelper; import org.whispersystems.textsecuregcm.tests.util.KeysHelper; import org.whispersystems.textsecuregcm.util.Pair; @@ -125,6 +126,9 @@ void setUp() { final SecureValueRecovery2Client svr2Client = mock(SecureValueRecovery2Client.class); when(svr2Client.deleteBackups(any())).thenReturn(CompletableFuture.completedFuture(null)); + final SecureValueRecovery3Client svr3Client = mock(SecureValueRecovery3Client.class); + when(svr3Client.deleteBackups(any())).thenReturn(CompletableFuture.completedFuture(null)); + final PhoneNumberIdentifiers phoneNumberIdentifiers = new PhoneNumberIdentifiers(DYNAMO_DB_EXTENSION.getDynamoDbAsyncClient(), DynamoDbExtensionSchema.Tables.PNI.tableName()); @@ -157,6 +161,7 @@ void setUp() { profilesManager, secureStorageClient, svr2Client, + svr3Client, mock(DisconnectionRequestManager.class), mock(RegistrationRecoveryPasswordsManager.class), clientPublicKeysManager, diff --git a/service/src/test/resources/config/test.yml b/service/src/test/resources/config/test.yml index a12f1b150..9a187070b 100644 --- a/service/src/test/resources/config/test.yml +++ b/service/src/test/resources/config/test.yml @@ -213,34 +213,36 @@ svr2: -----END CERTIFICATE----- svr3: - uri: svr3.example.com + backend1Uri: backend1.example.com + backend2Uri: backend2.example.com + backend3Uri: backend3.example.com userAuthenticationTokenSharedSecret: secret://svr3.userAuthenticationTokenSharedSecret userIdTokenSharedSecret: secret://svr3.userIdTokenSharedSecret svrCaCertificates: + # This is a randomly generated test certificate - | -----BEGIN CERTIFICATE----- - ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz - ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz - ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz - ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz - ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz - ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz - ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz - ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz - ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz - ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz - ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz - ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz - ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz - ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz - ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz - ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz - ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz - ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz - AAAAAAAAAAAAAAAAAAAA + MIIDazCCAlOgAwIBAgIUW5lcNWkuynRVc8Rq5pO6mHQBuZAwDQYJKoZIhvcNAQEL + BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM + GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yNDAzMjUwMzE4MTNaFw0yOTAz + MjQwMzE4MTNaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw + HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB + AQUAA4IBDwAwggEKAoIBAQCfH4Um+fv2r4KudhD37/UXp8duRLTmp4XvpBTpDHpD + 2HF8p2yThVKlJnMkP/9Ey1Rb0vhxO7DCltLdW8IYcxJuHoyMvyhGUEtxxkOZbrk8 + ciUR9jTZ37x7vXRGj/RxcdlS6iD0MeF0D/LAkImt4T/kiKwDbENrVEnYWJmipCKP + ribxWky7HqxDCoYMQr0zatxB3A9mx5stH+H3kbw3CZcm+ugF9ZIKDEVHb0lf28gq + llmD120q/vs9YV3rzVL7sBGDqf6olkulvHQJKElZg2rdcHWFcngSlU2BjR04oyuH + c/SSiLSB3YB0tdFGta5uorXyV1y7RElPeBfOfvEjsG3TAgMBAAGjUzBRMB0GA1Ud + DgQWBBQX+xlgSWWbDjv0SrJ+h67xauJ80zAfBgNVHSMEGDAWgBQX+xlgSWWbDjv0 + SrJ+h67xauJ80zAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAw + ZG2MCCjscn6h/QOoJU+IDfa68OqLq0I37gMnLMde4yEhAmm//miePIq4Uz9GRJ+h + rAmdEnspKgyQ93PjF7Xpk/JdJA4B1bIrsOl/cSwqx2sFhRt8Kt1DHGlGWXqOaHRP + UkZ86MyRL3sXly6WkxEYxZJeQaOzMy2XmQh7grzrlTBuSI+0xf7vsRRDipxr6LVQ + 6qGWyGODLLc2JD1IXj/1HpRVT2LoGGlKMuyxACQAm4oak1vvJ9mGxgfd9AU+eo58 + O/esB2Eaf+QqMPELdFSZQfG2jvp+3WQTZK8fDKHyLr076G3UetEMy867F6fzTSZd + 9Kxq0DY7RCEpdHMCKcOL -----END CERTIFICATE----- - messageCache: # Redis server configuration for message store cache persistDelayMinutes: 1 cluster: