From a88560e557d1c26a9fa451d396e0cf86877771e8 Mon Sep 17 00:00:00 2001 From: Ravi Khadiwala Date: Tue, 28 May 2024 13:41:14 -0500 Subject: [PATCH] Add gRPC backup services --- .../textsecuregcm/WhisperServerService.java | 5 + .../grpc/BackupsAnonymousGrpcService.java | 226 ++++++++ .../grpc/BackupsGrpcService.java | 122 ++++ .../main/proto/org/signal/chat/backups.proto | 544 ++++++++++++++++++ .../main/proto/org/signal/chat/common.proto | 12 + .../grpc/BackupsAnonymousGrpcServiceTest.java | 328 +++++++++++ .../grpc/BackupsGrpcServiceTest.java | 243 ++++++++ .../textsecuregcm/grpc/GrpcTestUtils.java | 4 +- 8 files changed, 1482 insertions(+), 2 deletions(-) create mode 100644 service/src/main/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcService.java create mode 100644 service/src/main/java/org/whispersystems/textsecuregcm/grpc/BackupsGrpcService.java create mode 100644 service/src/main/proto/org/signal/chat/backups.proto create mode 100644 service/src/test/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcServiceTest.java create mode 100644 service/src/test/java/org/whispersystems/textsecuregcm/grpc/BackupsGrpcServiceTest.java diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java index e57ee20d4..dbcd76602 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java @@ -154,6 +154,7 @@ import org.whispersystems.textsecuregcm.grpc.ProfileAnonymousGrpcService; import org.whispersystems.textsecuregcm.grpc.ProfileGrpcService; import org.whispersystems.textsecuregcm.grpc.RequestAttributesInterceptor; +import org.whispersystems.textsecuregcm.grpc.ValidatingInterceptor; import org.whispersystems.textsecuregcm.grpc.net.GrpcClientConnectionManager; import org.whispersystems.textsecuregcm.grpc.net.ManagedDefaultEventLoopGroup; import org.whispersystems.textsecuregcm.grpc.net.ManagedLocalGrpcServer; @@ -862,6 +863,8 @@ public void run(WhisperServerConfiguration config, Environment environment) thro final RequestAttributesInterceptor requestAttributesInterceptor = new RequestAttributesInterceptor(grpcClientConnectionManager); + final ValidatingInterceptor validatingInterceptor = new ValidatingInterceptor(); + final LocalAddress anonymousGrpcServerAddress = new LocalAddress("grpc-anonymous"); final LocalAddress authenticatedGrpcServerAddress = new LocalAddress("grpc-authenticated"); @@ -875,6 +878,7 @@ protected void configureServer(final ServerBuilder serverBuilder) { .intercept( new ExternalRequestFilter(config.getExternalRequestFilterConfiguration().permittedInternalRanges(), config.getExternalRequestFilterConfiguration().grpcMethods())) + .intercept(validatingInterceptor) // TODO: specialize metrics with user-agent platform .intercept(metricCollectingServerInterceptor) .intercept(errorMappingInterceptor) @@ -897,6 +901,7 @@ protected void configureServer(final ServerBuilder serverBuilder) { // http://grpc.github.io/grpc-java/javadoc/io/grpc/ServerBuilder.html#intercept-io.grpc.ServerInterceptor- serverBuilder // TODO: specialize metrics with user-agent platform + .intercept(validatingInterceptor) .intercept(metricCollectingServerInterceptor) .intercept(errorMappingInterceptor) .intercept(remoteDeprecationFilter) diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcService.java b/service/src/main/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcService.java new file mode 100644 index 000000000..c785493df --- /dev/null +++ b/service/src/main/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcService.java @@ -0,0 +1,226 @@ +/* + * Copyright 2024 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ +package org.whispersystems.textsecuregcm.grpc; + +import com.google.protobuf.ByteString; +import io.grpc.Status; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import org.signal.chat.backup.CopyMediaRequest; +import org.signal.chat.backup.CopyMediaResponse; +import org.signal.chat.backup.DeleteAllRequest; +import org.signal.chat.backup.DeleteAllResponse; +import org.signal.chat.backup.DeleteMediaRequest; +import org.signal.chat.backup.DeleteMediaResponse; +import org.signal.chat.backup.GetBackupInfoRequest; +import org.signal.chat.backup.GetBackupInfoResponse; +import org.signal.chat.backup.GetCdnCredentialsRequest; +import org.signal.chat.backup.GetCdnCredentialsResponse; +import org.signal.chat.backup.GetUploadFormRequest; +import org.signal.chat.backup.GetUploadFormResponse; +import org.signal.chat.backup.ListMediaRequest; +import org.signal.chat.backup.ListMediaResponse; +import org.signal.chat.backup.ReactorBackupsAnonymousGrpc; +import org.signal.chat.backup.RefreshRequest; +import org.signal.chat.backup.RefreshResponse; +import org.signal.chat.backup.SetPublicKeyRequest; +import org.signal.chat.backup.SetPublicKeyResponse; +import org.signal.chat.backup.SignedPresentation; +import org.signal.libsignal.protocol.InvalidKeyException; +import org.signal.libsignal.protocol.ecc.ECPublicKey; +import org.signal.libsignal.zkgroup.InvalidInputException; +import org.signal.libsignal.zkgroup.backups.BackupAuthCredentialPresentation; +import org.whispersystems.textsecuregcm.auth.AuthenticatedBackupUser; +import org.whispersystems.textsecuregcm.backup.BackupManager; +import org.whispersystems.textsecuregcm.backup.CopyParameters; +import org.whispersystems.textsecuregcm.backup.MediaEncryptionParameters; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +public class BackupsAnonymousGrpcService extends ReactorBackupsAnonymousGrpc.BackupsAnonymousImplBase { + + private final BackupManager backupManager; + + public BackupsAnonymousGrpcService(final BackupManager backupManager) { + this.backupManager = backupManager; + } + + @Override + public Mono getCdnCredentials(final GetCdnCredentialsRequest request) { + return authenticateBackupUserMono(request.getSignedPresentation()) + .map(user -> backupManager.generateReadAuth(user, request.getCdn())) + .map(credentials -> GetCdnCredentialsResponse.newBuilder().putAllHeaders(credentials).build()); + } + + @Override + public Mono getBackupInfo(final GetBackupInfoRequest request) { + return Mono.fromFuture(() -> + authenticateBackupUser(request.getSignedPresentation()).thenCompose(backupManager::backupInfo)) + .map(info -> GetBackupInfoResponse.newBuilder() + .setBackupName(info.messageBackupKey()) + .setCdn(info.cdn()) + .setBackupDir(info.backupSubdir()) + .setMediaDir(info.mediaSubdir()) + .setUsedSpace(info.mediaUsedSpace().orElse(0L)) + .build()); + } + + @Override + public Mono refresh(final RefreshRequest request) { + return Mono.fromFuture(() -> authenticateBackupUser(request.getSignedPresentation()) + .thenCompose(backupManager::ttlRefresh)) + .thenReturn(RefreshResponse.getDefaultInstance()); + } + + @Override + public Mono setPublicKey(final SetPublicKeyRequest request) { + final ECPublicKey publicKey = deserialize(ECPublicKey::new, request.getPublicKey().toByteArray()); + final BackupAuthCredentialPresentation presentation = deserialize( + BackupAuthCredentialPresentation::new, + request.getSignedPresentation().getPresentation().toByteArray()); + final byte[] signature = request.getSignedPresentation().getPresentationSignature().toByteArray(); + + return Mono.fromFuture(() -> backupManager.setPublicKey(presentation, signature, publicKey)) + .thenReturn(SetPublicKeyResponse.getDefaultInstance()); + } + + + @Override + public Mono getUploadForm(final GetUploadFormRequest request) { + return authenticateBackupUserMono(request.getSignedPresentation()) + .flatMap(backupUser -> switch (request.getUploadTypeCase()) { + case MESSAGES -> Mono.fromFuture(backupManager.createMessageBackupUploadDescriptor(backupUser)); + case MEDIA -> Mono.fromCompletionStage(backupManager.createTemporaryAttachmentUploadDescriptor(backupUser)); + case UPLOADTYPE_NOT_SET -> Mono.error(Status.INVALID_ARGUMENT + .withDescription("Must set upload_type") + .asRuntimeException()); + }) + .map(uploadDescriptor -> GetUploadFormResponse.newBuilder() + .setCdn(uploadDescriptor.cdn()) + .setKey(uploadDescriptor.key()) + .setSignedUploadLocation(uploadDescriptor.signedUploadLocation()) + .putAllHeaders(uploadDescriptor.headers()) + .build()); + } + + @Override + public Flux copyMedia(final CopyMediaRequest request) { + return authenticateBackupUserMono(request.getSignedPresentation()) + .flatMapMany(backupUser -> backupManager.copyToBackup(backupUser, + request.getItemsList().stream().map(item -> new CopyParameters( + item.getSourceAttachmentCdn(), item.getSourceKey(), + // uint32 in proto, make sure it fits in a signed int + fromUnsignedExact(item.getObjectLength()), + new MediaEncryptionParameters(item.getEncryptionKey().toByteArray(), item.getHmacKey().toByteArray()), + item.getMediaId().toByteArray())).toList())) + .map(copyResult -> { + CopyMediaResponse.Builder builder = CopyMediaResponse + .newBuilder() + .setMediaId(ByteString.copyFrom(copyResult.mediaId())); + builder = switch (copyResult.outcome()) { + case SUCCESS -> builder + .setSuccess(CopyMediaResponse.CopySuccess.newBuilder().setCdn(copyResult.cdn()).build()); + case OUT_OF_QUOTA -> builder + .setOutOfSpace(CopyMediaResponse.OutOfSpace.getDefaultInstance()); + case SOURCE_WRONG_LENGTH -> builder + .setWrongSourceLength(CopyMediaResponse.WrongSourceLength.getDefaultInstance()); + case SOURCE_NOT_FOUND -> builder + .setSourceNotFound(CopyMediaResponse.SourceNotFound.getDefaultInstance()); + }; + return builder.build(); + }); + + } + + @Override + public Mono listMedia(final ListMediaRequest request) { + return authenticateBackupUserMono(request.getSignedPresentation()).zipWhen( + backupUser -> Mono.fromFuture(backupManager.list( + backupUser, + request.hasCursor() ? Optional.of(request.getCursor()) : Optional.empty(), + request.getLimit()).toCompletableFuture()), + + (backupUser, listResult) -> { + final ListMediaResponse.Builder builder = ListMediaResponse.newBuilder(); + for (BackupManager.StorageDescriptorWithLength sd : listResult.media()) { + builder.addPage(ListMediaResponse.ListEntry.newBuilder() + .setMediaId(ByteString.copyFrom(sd.key())) + .setCdn(sd.cdn()) + .setLength(sd.length()) + .build()); + } + builder + .setBackupDir(backupUser.backupDir()) + .setMediaDir(backupUser.mediaDir()); + listResult.cursor().ifPresent(builder::setCursor); + return builder.build(); + }); + + } + + @Override + public Mono deleteAll(final DeleteAllRequest request) { + return Mono.fromFuture(() -> authenticateBackupUser(request.getSignedPresentation()) + .thenCompose(backupManager::deleteEntireBackup)) + .thenReturn(DeleteAllResponse.getDefaultInstance()); + } + + @Override + public Flux deleteMedia(final DeleteMediaRequest request) { + return Mono + .fromFuture(() -> authenticateBackupUser(request.getSignedPresentation())) + .flatMapMany(backupUser -> backupManager.deleteMedia(backupUser, request + .getItemsList() + .stream() + .map(item -> new BackupManager.StorageDescriptor(item.getCdn(), item.getMediaId().toByteArray())) + .toList())) + .map(storageDescriptor -> DeleteMediaResponse.newBuilder() + .setMediaId(ByteString.copyFrom(storageDescriptor.key())) + .setCdn(storageDescriptor.cdn()).build()); + } + + private Mono authenticateBackupUserMono(final SignedPresentation signedPresentation) { + return Mono.fromFuture(() -> authenticateBackupUser(signedPresentation)); + } + + private CompletableFuture authenticateBackupUser( + final SignedPresentation signedPresentation) { + if (signedPresentation == null) { + throw Status.UNAUTHENTICATED.asRuntimeException(); + } + try { + return backupManager.authenticateBackupUser( + new BackupAuthCredentialPresentation(signedPresentation.getPresentation().toByteArray()), + signedPresentation.getPresentationSignature().toByteArray()); + } catch (InvalidInputException e) { + throw Status.UNAUTHENTICATED.withDescription("Could not deserialize presentation").asRuntimeException(); + } + } + + /** + * Convert an int from a proto uint32 to a signed positive integer, throwing if the value exceeds + * {@link Integer#MAX_VALUE}. To convert to a long, see {@link Integer#toUnsignedLong(int)} + */ + private static int fromUnsignedExact(final int i) { + if (i < 0) { + throw Status.INVALID_ARGUMENT.withDescription("Invalid size").asRuntimeException(); + } + return i; + } + + private interface Deserializer { + + T deserialize(byte[] bytes) throws InvalidInputException, InvalidKeyException; + } + + private static T deserialize(Deserializer deserializer, byte[] bytes) { + try { + return deserializer.deserialize(bytes); + } catch (InvalidInputException | InvalidKeyException e) { + throw Status.INVALID_ARGUMENT.withDescription("Invalid serialization").asRuntimeException(); + } + } + +} diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/grpc/BackupsGrpcService.java b/service/src/main/java/org/whispersystems/textsecuregcm/grpc/BackupsGrpcService.java new file mode 100644 index 000000000..605da9fba --- /dev/null +++ b/service/src/main/java/org/whispersystems/textsecuregcm/grpc/BackupsGrpcService.java @@ -0,0 +1,122 @@ +/* + * Copyright 2024 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ +package org.whispersystems.textsecuregcm.grpc; + +import com.google.protobuf.ByteString; +import io.grpc.Status; +import java.time.Instant; +import java.util.List; +import java.util.stream.Collectors; +import org.signal.chat.backup.GetBackupAuthCredentialsRequest; +import org.signal.chat.backup.GetBackupAuthCredentialsResponse; +import org.signal.chat.backup.ReactorBackupsGrpc; +import org.signal.chat.backup.RedeemReceiptRequest; +import org.signal.chat.backup.RedeemReceiptResponse; +import org.signal.chat.backup.SetBackupIdRequest; +import org.signal.chat.backup.SetBackupIdResponse; +import org.signal.chat.common.ZkCredential; +import org.signal.libsignal.zkgroup.InvalidInputException; +import org.signal.libsignal.zkgroup.backups.BackupAuthCredentialRequest; +import org.signal.libsignal.zkgroup.backups.BackupCredentialType; +import org.signal.libsignal.zkgroup.receipts.ReceiptCredentialPresentation; +import org.whispersystems.textsecuregcm.auth.grpc.AuthenticatedDevice; +import org.whispersystems.textsecuregcm.auth.grpc.AuthenticationUtil; +import org.whispersystems.textsecuregcm.backup.BackupAuthManager; +import org.whispersystems.textsecuregcm.storage.Account; +import org.whispersystems.textsecuregcm.storage.AccountsManager; +import reactor.core.publisher.Mono; + +public class BackupsGrpcService extends ReactorBackupsGrpc.BackupsImplBase { + + private final AccountsManager accountManager; + private final BackupAuthManager backupAuthManager; + + public BackupsGrpcService(final AccountsManager accountManager, final BackupAuthManager backupAuthManager) { + this.accountManager = accountManager; + this.backupAuthManager = backupAuthManager; + } + + + @Override + public Mono setBackupId(SetBackupIdRequest request) { + + final BackupAuthCredentialRequest messagesCredentialRequest = deserialize( + BackupAuthCredentialRequest::new, + request.getMessagesBackupAuthCredentialRequest().toByteArray()); + + final BackupAuthCredentialRequest mediaCredentialRequest = deserialize( + BackupAuthCredentialRequest::new, + request.getMediaBackupAuthCredentialRequest().toByteArray()); + + return authenticatedAccount() + .flatMap(account -> Mono.fromFuture( + backupAuthManager.commitBackupId(account, messagesCredentialRequest, mediaCredentialRequest))) + .thenReturn(SetBackupIdResponse.getDefaultInstance()); + } + + public Mono redeemReceipt(RedeemReceiptRequest request) { + final ReceiptCredentialPresentation receiptCredentialPresentation = deserialize( + ReceiptCredentialPresentation::new, + request.getPresentation().toByteArray()); + return authenticatedAccount() + .flatMap(account -> Mono.fromFuture(backupAuthManager.redeemReceipt(account, receiptCredentialPresentation))) + .thenReturn(RedeemReceiptResponse.getDefaultInstance()); + } + + @Override + public Mono getBackupAuthCredentials(GetBackupAuthCredentialsRequest request) { + return authenticatedAccount().flatMap(account -> { + final Mono> messageCredentials = Mono.fromCompletionStage(() -> + backupAuthManager.getBackupAuthCredentials( + account, + BackupCredentialType.MESSAGES, + Instant.ofEpochSecond(request.getRedemptionStart()), + Instant.ofEpochSecond(request.getRedemptionStop()))); + final Mono> mediaCredentials = Mono.fromCompletionStage(() -> + backupAuthManager.getBackupAuthCredentials( + account, + BackupCredentialType.MEDIA, + Instant.ofEpochSecond(request.getRedemptionStart()), + Instant.ofEpochSecond(request.getRedemptionStop()))); + + return messageCredentials.zipWith(mediaCredentials, (messageCreds, mediaCreds) -> + GetBackupAuthCredentialsResponse.newBuilder() + .putAllMessageCredentials(messageCreds.stream().collect(Collectors.toMap( + c -> c.redemptionTime().getEpochSecond(), + c -> ZkCredential.newBuilder() + .setCredential(ByteString.copyFrom(c.credential().serialize())) + .setRedemptionTime(c.redemptionTime().getEpochSecond()) + .build()))) + .putAllMediaCredentials(mediaCreds.stream().collect(Collectors.toMap( + c -> c.redemptionTime().getEpochSecond(), + c -> ZkCredential.newBuilder() + .setCredential(ByteString.copyFrom(c.credential().serialize())) + .setRedemptionTime(c.redemptionTime().getEpochSecond()) + .build()))) + .build()); + }); + } + + private Mono authenticatedAccount() { + final AuthenticatedDevice authenticatedDevice = AuthenticationUtil.requireAuthenticatedDevice(); + return Mono + .fromFuture(() -> accountManager.getByAccountIdentifierAsync(authenticatedDevice.accountIdentifier())) + .map(maybeAccount -> maybeAccount.orElseThrow(Status.UNAUTHENTICATED::asRuntimeException)); + } + + private interface Deserializer { + + T deserialize(byte[] bytes) throws InvalidInputException; + } + + private T deserialize(Deserializer deserializer, byte[] bytes) { + try { + return deserializer.deserialize(bytes); + } catch (InvalidInputException e) { + throw Status.INVALID_ARGUMENT.withDescription("Invalid serialization").asRuntimeException(); + } + } + +} diff --git a/service/src/main/proto/org/signal/chat/backups.proto b/service/src/main/proto/org/signal/chat/backups.proto new file mode 100644 index 000000000..636a9b7f2 --- /dev/null +++ b/service/src/main/proto/org/signal/chat/backups.proto @@ -0,0 +1,544 @@ +/* + * Copyright 2024 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +syntax = "proto3"; + +option java_multiple_files = true; + +package org.signal.chat.backup; + +import "google/protobuf/empty.proto"; +import "org/signal/chat/common.proto"; +import "org/signal/chat/require.proto"; + +/** + * Service for backup operations that require account authentication. + * + * Most actual backup operations operate on the backup-id and cannot be linked + * to the caller's account, but setting up anonymous credentials and changing + * backup tier requires account authentication. + * + * All rpcs on this service may return these errors. rpc specific errors + * documented on the individual rpc. + * + * errors: + * UNAUTHENTICATED Authentication failed or the account does not exist + * INVALID_ARGUMENT The request did not meet a documented requirement + * RESOURCE_EXHAUSTED Rate limit exceeded. A retry-after header containing an + * ISO8601 duration string will be present in the response + * trailers. + */ +service Backups { + option (require.auth) = AUTH_ONLY_AUTHENTICATED; + + /** + * Set a (blinded) backup-id for the account. + * + * Each account may have a single active backup-id that can be used + * to store and retrieve backups. Once the backup-id is set, + * BackupAuthCredentials can be generated using GetBackupAuthCredentials. + * + * The blinded backup-id and the key-pair used to blind it must be derived + * from a recoverable secret. + * + * errors: + * PERMISSION_DENIED: This account is not currently eligible for backups + */ + rpc SetBackupId(SetBackupIdRequest) returns (SetBackupIdResponse) {} + + /** + * Redeem a receipt acquired from /v1/subscription/{subscriberId}/receipt_credentials + * to mark the account as eligible for the paid backup tier. + * + * After successful redemption, subsequent requests to + * GetBackupAuthCredentials will return credentials with the level on the + * provided receipt until the expiration time on the receipt. + */ + rpc RedeemReceipt(RedeemReceiptRequest) returns (RedeemReceiptResponse) {} + + /** + * After setting a blinded backup-id with PUT /v1/archives/, this fetches + * credentials that can be used to perform operations against that backup-id. + * Clients may (and should) request up to 7 days of credentials at a time. + * + * The redemption_start and redemption_end seconds must be UTC day aligned, and + * must not span more than 7 days. + * + * Each credential contains a receipt level which indicates the backup level + * the credential is good for. If the account has paid backup access that + * expires at some point in the provided redemption window, credentials with + * redemption times after the expiration may be on a lower backup level. + * + * Clients must validate the receipt level on the credential matches a known + * receipt level before using it. + * + * errors: + * NOT_FOUND: Could not find an existing blinded backup id associated with the + * account. + */ + rpc GetBackupAuthCredentials(GetBackupAuthCredentialsRequest) returns (GetBackupAuthCredentialsResponse) {} +} + +message SetBackupIdRequest { + /** + * A BackupAuthCredentialRequest containing a blinded encrypted backup-id, + * encoded in standard padded base64. This backup-id should be used for + * message backups only, and must have the message backup type set on the + * credential. + */ + bytes messages_backup_auth_credential_request = 1; + + /** + * A BackupAuthCredentialRequest containing a blinded encrypted backup-id, + * encoded in standard padded base64. This backup-id should be used for + * media only, and must have the media type set on the credential. + */ + bytes media_backup_auth_credential_request = 2; +} +message SetBackupIdResponse {} + + +message RedeemReceiptRequest { + /** + * Presentation for a previously acquired receipt, serialized with libsignal + */ + bytes presentation = 1; +} +message RedeemReceiptResponse {} + +message GetBackupAuthCredentialsRequest { + /** + * The redemption time for the first credential. This must be a day-aligned + * seconds since epoch in UTC. + */ + int64 redemption_start = 1 [(require.range).min = 1]; + + /** + * The redemption time for the last credential. This must be a day-aligned + * seconds since epoch in UTC. The span between redemptionStart and + * redemptionEnd must not exceed 7 days. + */ + int64 redemption_stop = 2 [(require.range).min = 1]; +} + +message GetBackupAuthCredentialsResponse { + /** + * The requested message backup ZkCredentials indexed by the start of their + * validity period. The smallest key should be for the requested + * redemption_start, the largest for the requested the requested + * redemption_end. + */ + map message_credentials = 1; + + /** + * The requested media backup ZkCredentials indexed by the start of their + * validity period. The smallest key should be for the requested + * redemption_start, the largest for the requested the requested + * redemption_end. + */ + map media_credentials = 2; +} + +/** + * Service for backup operations with anonymous credentials + * + * This service never requires account authentication. It instead requires a + * backup-id authenticated with an anonymous credential that cannot be linked + * to the account. + * + * To register an anonymous credential: + * 1. Set a backup-id on the authenticated channel via Backups::SetBackupId + * 2. Retrieve BackupAuthCredentials via Backups::GetBackupAuthCredentials + * 3. Generate a key pair and set the public key via + * BackupsAnonymous::SetPublicKey + * + * Unless otherwise noted, requests for this service require a + * SignedPresentation, which includes: + * - a presentation generated from a BackupAuthCredential issued by + * GetBackupAuthCredentials + * - a signature of that presentation using the private key of a key pair + * previously set with SetPublicKey. + * + * All RPCs on this service may return these errors. RPC specific errors + * documented on the individual RPC. + * + * errors: + * UNAUTHENTICATED Either the presentation was missing, the credential was + * expired, presentation verification failed, the signature + * was incorrect, there was no committed public key for the + * corresponding backup id, or the request was made on a + * non-anonymous channel. + * PERMISSION_DENIED The credential does not have permission to perform the + * requested action. + * RESOURCE_EXHAUSTED Rate limit exceeded. A retry-after header containing an + * ISO8601 duration string will be present in the response + * trailers. + * INVALID_ARGUMENT The request did not meet a documented requirement + */ +service BackupsAnonymous { + option (require.auth) = AUTH_ONLY_ANONYMOUS; + + /** + * Retrieve credentials used to read objects stored on the backup cdn + */ + rpc GetCdnCredentials(GetCdnCredentialsRequest) returns (GetCdnCredentialsResponse) {} + + /** + * Retrieve information about the currently stored backup + */ + rpc GetBackupInfo(GetBackupInfoRequest) returns (GetBackupInfoResponse) {} + + /** + * Permanently set the public key of an ED25519 key-pair for the backup-id. + * All requests (including this one!) must sign their BackupAuthCredential + * presentations with the private key corresponding to the provided public key. + * + * Trying to set a public key when a different one is already set will return + * an UNAUTHENTICATED error. + */ + rpc SetPublicKey(SetPublicKeyRequest) returns (SetPublicKeyResponse) {} + + /** + * Refresh the backup, indicating that the backup is still active. Clients + * must periodically upload new backups or perform a refresh. If a backup has + * not been active for 30 days, it may deleted + */ + rpc Refresh(RefreshRequest) returns (RefreshResponse) {} + + /** + * Retrieve an upload form that can be used to perform a resumable upload + */ + rpc GetUploadForm(GetUploadFormRequest) returns (GetUploadFormResponse) {} + + /** + * Copy and re-encrypt media from the attachments cdn into the backup cdn. + * The original, already encrypted, attachments will be encrypted with the + * provided key material before being copied. + * + * The copy operation is not atomic and responses will be returned as copy + * operations complete with detailed information about the outcome. If an + * error is encountered, not all requests may be reflected in the responses. + * + * On retries, a particular destination media id must not be reused with a + * different source media id or different encryption parameters. + */ + rpc CopyMedia(CopyMediaRequest) returns (stream CopyMediaResponse) {} + + /** + * Retrieve a page of media objects stored for this backup-id. A client may + * have previously stored media objects that are no longer referenced in their + * current backup. To reclaim storage space used by these orphaned objects, + * perform a list operation and remove any unreferenced media objects + * via DeleteMedia. + */ + rpc ListMedia(ListMediaRequest) returns (ListMediaResponse) {} + + /** + * Delete media objects stored with this backup-id. Streams the locations of + * media items back when the item has successfully been removed. + */ + rpc DeleteMedia(DeleteMediaRequest) returns (stream DeleteMediaResponse) {} + + /** + * Delete all backup metadata, objects, and stored public key. To use + * backups again, a public key must be resupplied. + */ + rpc DeleteAll(DeleteAllRequest) returns (DeleteAllResponse) {} +} + +message SignedPresentation { + /** + * Presentation of a BackupAuthCredential previously retrieved from + * GetBackupAuthCredentials on the authenticated channel + */ + bytes presentation = 1; + + /** + * The presentation signed with the private key corresponding to the public + * key set with SetPublicKey + */ + bytes presentation_signature = 2; +} + +message SetPublicKeyRequest { + SignedPresentation signed_presentation = 1; + + /** + * The public key, serialized in libsignal's elliptic-curve public key format. + */ + bytes public_key = 2; +} +message SetPublicKeyResponse {} + +message GetCdnCredentialsRequest { + SignedPresentation signed_presentation = 1; + int32 cdn = 2; +} +message GetCdnCredentialsResponse { + /** + * Headers to include with requests to the read from the backup CDN. Includes + * time limited read-only credentials. + */ + map headers = 1; +} + +message GetBackupInfoRequest { + SignedPresentation signed_presentation = 1; +} +message GetBackupInfoResponse { + /** + * The base directory of your backup data on the cdn. The message backup can + * be found in the returned cdn at /backup_dir/backup_name and stored media can + * be found at /backup_dir/media_dir/media_id + */ + string backup_dir = 1; + + /** + * The prefix path component for media objects on a cdn. Stored media for a + * media_id can be found at /backup_dir/media_dir/media_id, where the media_id + * is encoded in unpadded url-safe base64. + */ + string media_dir = 2; + + /** + * The CDN type where the message backup is stored. Media may be stored + * elsewhere. If absent, no message backup currently exists. + */ + optional int32 cdn = 3; + + /** + * The name of the most recent message backup on the cdn. The backup is at + * /backup_dir/backup_name. If absent, no message backup currently exists. + */ + optional string backup_name = 4; + + /** + * The amount of space used to store media + */ + uint64 used_space = 5; +} + +message RefreshRequest { + SignedPresentation signed_presentation = 1; +} +message RefreshResponse { + SignedPresentation signed_presentation = 1; +} + +message GetUploadFormRequest { + SignedPresentation signed_presentation = 1; + + message MessagesUploadType {} + message MediaUploadType {} + oneof upload_type { + /** + * Retrieve an upload form that can be used to perform a resumable upload of + * a message backup. The finished upload will be available on the backup cdn. + */ + MessagesUploadType messages = 2; + + /** + * Retrieve an upload form for a temporary location that can be used to + * perform a resumable upload of an attachment. After uploading, the + * attachment can be copied into the backup via CopyMedia. + * + * Behaves identically to the account authenticated version at /attachments. + */ + MediaUploadType media = 3; + } +} +message GetUploadFormResponse { + /** + * Indicates the CDN type. 3 indicates resumable uploads using TUS + */ + int32 cdn = 1; + + /** + * The location within the specified cdn where the finished upload can be found + */ + string key = 2; + + /** + * A map of headers to include with all upload requests. Potentially contains + * time-limited upload credentials + */ + map headers = 3; + + /** + * The URL to upload to with the appropriate protocol + */ + string signed_upload_location = 4; +} + +message CopyMediaItem { + /** + * The attachment cdn of the object to copy into the backup + */ + int32 source_attachment_cdn = 1 [(require.present) = true]; + + /** + * The attachment key of the object to copy into the backup + */ + string source_key = 2 [(require.nonEmpty) = true]; + + /** + * The length of the source attachment before the encryption applied by the + * copy operation + */ + uint32 object_length = 3; + + /** + * media_id to copy on to the backup CDN + */ + bytes media_id = 4 [(require.exactlySize) = 15]; + + /** + * A 32-byte key for the MAC + */ + bytes hmac_key = 5 [(require.exactlySize) = 32]; + + /** + * A 32-byte encryption key for AES + */ + bytes encryption_key = 6 [(require.exactlySize) = 32]; +} + +message CopyMediaRequest { + SignedPresentation signed_presentation = 1; + + /** + * Items to copy + */ + repeated CopyMediaItem items = 2; +} + +message CopyMediaResponse { + message SourceNotFound {} + message WrongSourceLength {} + message OutOfSpace {} + message CopySuccess { + /** + * The backup cdn where this media object is stored + */ + int32 cdn = 1; + } + + /** + * The 15-byte media_id from the corresponding CopyMediaItem in the request + */ + bytes media_id = 1; + + oneof outcome { + /** + * The media item was successfully copied into the backup + */ + CopySuccess success = 2; + + /** + * The source object was not found + */ + SourceNotFound source_not_found = 3; + + /** + * The provided object length was incorrect + */ + WrongSourceLength wrong_source_length = 4; + + /** + * All media capacity has been consumed. Free some space to continue. + */ + OutOfSpace out_of_space = 5; + } +} + +message ListMediaRequest { + SignedPresentation signed_presentation = 1; + + /** + * A cursor returned by a previous call to ListMedia, absent on the first call + */ + optional string cursor = 2; + + /** + * If provided, the maximum number of entries to return in a page + */ + uint32 limit = 3 [(require.range) = {min: 0, max: 10000}]; +} +message ListMediaResponse { + message ListEntry { + /** + * The backup cdn where this media object is stored + */ + int32 cdn = 1; + /** + * The media_id of the object + */ + bytes media_id = 2; + /** + * The length of the object in bytes + */ + uint64 length = 3; + } + + /** + * A page of media objects stored for this backup ID + */ + repeated ListEntry page = 1; + + /** + * The base directory of the backup data on the cdn. The stored media can be + * found at /backup_dir/media_dir/media_id, where the media_id is encoded with + * unpadded url-safe base64. + */ + string backup_dir = 2; + + /** + * The prefix path component for the media objects. The stored media for + * media_id can be found at /backup_dir/media_dir/media_id, where the media_id + * is encoded with unpadded url-safe base64. + */ + string media_dir = 3; + + /** + * If set, the cursor value to pass to the next list request to continue + * listing. If absent, all objects have been listed + */ + optional string cursor = 4; +} + +message DeleteAllRequest { + SignedPresentation signed_presentation = 1; +} +message DeleteAllResponse {} + +message DeleteMediaItem { + /** + * The backup cdn where this media object is stored + */ + int32 cdn = 1; + + /** + * The media_id of the object to delete + */ + bytes media_id = 2; +} + +message DeleteMediaRequest { + SignedPresentation signed_presentation = 1; + + repeated DeleteMediaItem items = 2; +} + +message DeleteMediaResponse { + /** + * The backup cdn where the media object was stored + */ + int32 cdn = 1; + + /** + * The media_id of the object that was successfully deleted + */ + bytes media_id = 3; +} diff --git a/service/src/main/proto/org/signal/chat/common.proto b/service/src/main/proto/org/signal/chat/common.proto index dd5e0881a..6c78988a5 100644 --- a/service/src/main/proto/org/signal/chat/common.proto +++ b/service/src/main/proto/org/signal/chat/common.proto @@ -101,3 +101,15 @@ enum DeviceCapability { DEVICE_CAPABILITY_VERSIONED_EXPIRATION_TIMER = 4; DEVICE_CAPABILITY_STORAGE_SERVICE_RECORD_KEY_ROTATION = 5; } + +message ZkCredential { + /* + * Day on which this credential can be redeemed, in UTC seconds since epoch + */ + int64 redemption_time = 1; + + /* + * The ZK credential, using libsignal's serialization + */ + bytes credential = 2; +} diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcServiceTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcServiceTest.java new file mode 100644 index 000000000..5b1e3681c --- /dev/null +++ b/service/src/test/java/org/whispersystems/textsecuregcm/grpc/BackupsAnonymousGrpcServiceTest.java @@ -0,0 +1,328 @@ +/* + * Copyright 2023 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.whispersystems.textsecuregcm.grpc; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import static org.assertj.core.api.Assertions.assertThatNoException; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; + +import com.google.protobuf.ByteString; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import java.time.Clock; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junitpioneer.jupiter.cartesian.CartesianTest; +import org.mockito.Mock; +import org.signal.chat.backup.BackupsAnonymousGrpc; +import org.signal.chat.backup.CopyMediaItem; +import org.signal.chat.backup.CopyMediaRequest; +import org.signal.chat.backup.CopyMediaResponse; +import org.signal.chat.backup.DeleteMediaItem; +import org.signal.chat.backup.DeleteMediaRequest; +import org.signal.chat.backup.GetBackupInfoRequest; +import org.signal.chat.backup.GetBackupInfoResponse; +import org.signal.chat.backup.GetCdnCredentialsRequest; +import org.signal.chat.backup.GetCdnCredentialsResponse; +import org.signal.chat.backup.GetUploadFormRequest; +import org.signal.chat.backup.GetUploadFormResponse; +import org.signal.chat.backup.ListMediaRequest; +import org.signal.chat.backup.ListMediaResponse; +import org.signal.chat.backup.SetPublicKeyRequest; +import org.signal.chat.backup.SignedPresentation; +import org.signal.libsignal.protocol.ecc.Curve; +import org.signal.libsignal.zkgroup.VerificationFailedException; +import org.signal.libsignal.zkgroup.backups.BackupAuthCredentialPresentation; +import org.signal.libsignal.zkgroup.backups.BackupCredentialType; +import org.signal.libsignal.zkgroup.backups.BackupLevel; +import org.whispersystems.textsecuregcm.auth.AuthenticatedBackupUser; +import org.whispersystems.textsecuregcm.backup.BackupAuthTestUtil; +import org.whispersystems.textsecuregcm.backup.BackupManager; +import org.whispersystems.textsecuregcm.backup.BackupUploadDescriptor; +import org.whispersystems.textsecuregcm.backup.CopyResult; +import org.whispersystems.textsecuregcm.controllers.RateLimitExceededException; +import org.whispersystems.textsecuregcm.util.TestRandomUtil; +import reactor.core.publisher.Flux; + +class BackupsAnonymousGrpcServiceTest extends + SimpleBaseGrpcTest { + + private final UUID aci = UUID.randomUUID(); + private final byte[] messagesBackupKey = TestRandomUtil.nextBytes(32); + private final BackupAuthTestUtil backupAuthTestUtil = new BackupAuthTestUtil(Clock.systemUTC()); + private final BackupAuthCredentialPresentation presentation = + presentation(backupAuthTestUtil, messagesBackupKey, aci); + + @Mock + private BackupManager backupManager; + + @Override + protected BackupsAnonymousGrpcService createServiceBeforeEachTest() { + return new BackupsAnonymousGrpcService(backupManager); + } + + @BeforeEach + void setup() { + when(backupManager.authenticateBackupUser(any(), any())) + .thenReturn(CompletableFuture.completedFuture( + backupUser(presentation.getBackupId(), BackupCredentialType.MESSAGES, BackupLevel.PAID))); + } + + @Test + void setPublicKey() { + when(backupManager.setPublicKey(any(), any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + assertThatNoException().isThrownBy(() -> unauthenticatedServiceStub().setPublicKey(SetPublicKeyRequest.newBuilder() + .setPublicKey(ByteString.copyFrom(Curve.generateKeyPair().getPublicKey().serialize())) + .setSignedPresentation(signedPresentation(presentation)) + .build())); + } + + @Test + void setBadPublicKey() { + when(backupManager.setPublicKey(any(), any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + assertThatExceptionOfType(StatusRuntimeException.class).isThrownBy(() -> + unauthenticatedServiceStub().setPublicKey(SetPublicKeyRequest.newBuilder() + .setPublicKey(ByteString.copyFromUtf8("aaaaa")) // Invalid public key + .setSignedPresentation(signedPresentation(presentation)) + .build())) + .extracting(ex -> ex.getStatus().getCode()) + .isEqualTo(Status.Code.INVALID_ARGUMENT); + } + + @Test + void setMissingPublicKey() { + assertThatExceptionOfType(StatusRuntimeException.class).isThrownBy(() -> + unauthenticatedServiceStub().setPublicKey(SetPublicKeyRequest.newBuilder() + // Missing public key + .setSignedPresentation(signedPresentation(presentation)) + .build())) + .extracting(ex -> ex.getStatus().getCode()) + .isEqualTo(Status.Code.INVALID_ARGUMENT); + } + + + @Test + void putMediaBatchSuccess() { + final byte[][] mediaIds = {TestRandomUtil.nextBytes(15), TestRandomUtil.nextBytes(15)}; + when(backupManager.copyToBackup(any(), any())) + .thenReturn(Flux.just( + new CopyResult(CopyResult.Outcome.SUCCESS, mediaIds[0], 1), + new CopyResult(CopyResult.Outcome.SUCCESS, mediaIds[1], 1))); + + final CopyMediaRequest request = CopyMediaRequest.newBuilder() + .setSignedPresentation(signedPresentation(presentation)) + .addItems(CopyMediaItem.newBuilder() + .setSourceAttachmentCdn(3) + .setSourceKey("abc") + .setObjectLength(100) + .setMediaId(ByteString.copyFrom(mediaIds[0])) + .setHmacKey(ByteString.copyFrom(TestRandomUtil.nextBytes(32))) + .setEncryptionKey(ByteString.copyFrom(TestRandomUtil.nextBytes(32))) + .build()) + .addItems(CopyMediaItem.newBuilder() + .setSourceAttachmentCdn(3) + .setSourceKey("def") + .setObjectLength(200) + .setMediaId(ByteString.copyFrom(mediaIds[1])) + .setHmacKey(ByteString.copyFrom(TestRandomUtil.nextBytes(32))) + .setEncryptionKey(ByteString.copyFrom(TestRandomUtil.nextBytes(32))) + .build()) + .build(); + + final Iterator it = unauthenticatedServiceStub().copyMedia(request); + + for (int i = 0; i < 2; i++) { + final CopyMediaResponse response = it.next(); + assertThat(response.getSuccess().getCdn()).isEqualTo(1); + assertThat(response.getMediaId().toByteArray()).isEqualTo(mediaIds[i]); + } + assertThat(it.hasNext()).isFalse(); + } + + @Test + void putMediaBatchPartialFailure() { + // Copy four different mediaIds, with a variety of success/failure outcomes + final byte[][] mediaIds = IntStream.range(0, 4).mapToObj(i -> TestRandomUtil.nextBytes(15)).toArray(byte[][]::new); + final CopyResult.Outcome[] outcomes = new CopyResult.Outcome[]{ + CopyResult.Outcome.SUCCESS, + CopyResult.Outcome.SOURCE_NOT_FOUND, + CopyResult.Outcome.SOURCE_WRONG_LENGTH, + CopyResult.Outcome.OUT_OF_QUOTA + }; + when(backupManager.copyToBackup(any(), any())) + .thenReturn(Flux.fromStream(IntStream.range(0, 4) + .mapToObj(i -> new CopyResult( + outcomes[i], + mediaIds[i], + outcomes[i] == CopyResult.Outcome.SUCCESS ? 1 : null)))); + + final CopyMediaRequest request = CopyMediaRequest.newBuilder() + .setSignedPresentation(signedPresentation(presentation)) + .addAllItems(Arrays.stream(mediaIds) + .map(mediaId -> CopyMediaItem.newBuilder() + .setSourceAttachmentCdn(3) + .setSourceKey("abc") + .setObjectLength(100) + .setMediaId(ByteString.copyFrom(mediaId)) + .setHmacKey(ByteString.copyFrom(TestRandomUtil.nextBytes(32))) + .setEncryptionKey(ByteString.copyFrom(TestRandomUtil.nextBytes(32))) + .build()) + .collect(Collectors.toList())) + .build(); + + final Iterator responses = unauthenticatedServiceStub().copyMedia(request); + + // Verify that we get the expected response for each mediaId + for (int i = 0; i < mediaIds.length; i++) { + final CopyMediaResponse response = responses.next(); + switch (outcomes[i]) { + case SUCCESS -> assertThat(response.getSuccess().getCdn()).isEqualTo(1); + case SOURCE_WRONG_LENGTH -> assertThat(response.getWrongSourceLength()).isNotNull(); + case OUT_OF_QUOTA -> assertThat(response.getOutOfSpace()).isNotNull(); + case SOURCE_NOT_FOUND -> assertThat(response.getSourceNotFound()).isNotNull(); + } + assertThat(response.getMediaId().toByteArray()).isEqualTo(mediaIds[i]); + } + } + + @Test + void getBackupInfo() { + when(backupManager.backupInfo(any())).thenReturn(CompletableFuture.completedFuture(new BackupManager.BackupInfo( + 1, "myBackupDir", "myMediaDir", "filename", Optional.empty()))); + + final GetBackupInfoResponse response = unauthenticatedServiceStub().getBackupInfo(GetBackupInfoRequest.newBuilder() + .setSignedPresentation(signedPresentation(presentation)) + .build()); + assertThat(response.getBackupDir()).isEqualTo("myBackupDir"); + assertThat(response.getBackupName()).isEqualTo("filename"); + assertThat(response.getCdn()).isEqualTo(1); + assertThat(response.getUsedSpace()).isEqualTo(0L); + } + + + @CartesianTest + void list( + @CartesianTest.Values(booleans = {true, false}) final boolean cursorProvided, + @CartesianTest.Values(booleans = {true, false}) final boolean cursorReturned) + throws VerificationFailedException { + + final byte[] mediaId = TestRandomUtil.nextBytes(15); + final Optional expectedCursor = cursorProvided ? Optional.of("myCursor") : Optional.empty(); + final Optional returnedCursor = cursorReturned ? Optional.of("newCursor") : Optional.empty(); + + final int limit = 17; + + when(backupManager.list(any(), eq(expectedCursor), eq(limit))) + .thenReturn(CompletableFuture.completedFuture(new BackupManager.ListMediaResult( + List.of(new BackupManager.StorageDescriptorWithLength(1, mediaId, 100)), + returnedCursor))); + + final ListMediaRequest.Builder request = ListMediaRequest.newBuilder() + .setSignedPresentation(signedPresentation(presentation)) + .setLimit(limit); + if (cursorProvided) { + request.setCursor("myCursor"); + } + + final ListMediaResponse response = unauthenticatedServiceStub().listMedia(request.build()); + assertThat(response.getPageCount()).isEqualTo(1); + assertThat(response.getPage(0).getLength()).isEqualTo(100); + assertThat(response.getPage(0).getMediaId().toByteArray()).isEqualTo(mediaId); + assertThat(response.hasCursor() ? response.getCursor() : null).isEqualTo(returnedCursor.orElse(null)); + + } + + @Test + void delete() { + final DeleteMediaRequest request = DeleteMediaRequest.newBuilder() + .setSignedPresentation(signedPresentation(presentation)) + .addAllItems(IntStream.range(0, 100).mapToObj(i -> + DeleteMediaItem.newBuilder() + .setCdn(3) + .setMediaId(ByteString.copyFrom(TestRandomUtil.nextBytes(15))) + .build()) + .toList()).build(); + + when(backupManager.deleteMedia(any(), any())) + .thenReturn(Flux.fromStream(request.getItemsList().stream() + .map(m -> new BackupManager.StorageDescriptor(m.getCdn(), m.getMediaId().toByteArray())))); + + final AtomicInteger count = new AtomicInteger(0); + unauthenticatedServiceStub().deleteMedia(request).forEachRemaining(i -> count.getAndIncrement()); + assertThat(count.get()).isEqualTo(100); + } + + @Test + void mediaUploadForm() { + when(backupManager.createTemporaryAttachmentUploadDescriptor(any())) + .thenReturn(CompletableFuture.completedFuture( + new BackupUploadDescriptor(3, "abc", Map.of("k", "v"), "example.org"))); + final GetUploadFormRequest request = GetUploadFormRequest.newBuilder() + .setMedia(GetUploadFormRequest.MediaUploadType.getDefaultInstance()) + .setSignedPresentation(signedPresentation(presentation)) + .build(); + + final GetUploadFormResponse uploadForm = unauthenticatedServiceStub().getUploadForm(request); + assertThat(uploadForm.getCdn()).isEqualTo(3); + assertThat(uploadForm.getKey()).isEqualTo("abc"); + assertThat(uploadForm.getHeadersMap()).containsExactlyEntriesOf(Map.of("k", "v")); + assertThat(uploadForm.getSignedUploadLocation()).isEqualTo("example.org"); + + // rate limit + when(backupManager.createTemporaryAttachmentUploadDescriptor(any())) + .thenReturn(CompletableFuture.failedFuture(new RateLimitExceededException(null))); + assertThatExceptionOfType(StatusRuntimeException.class) + .isThrownBy(() -> unauthenticatedServiceStub().getUploadForm(request)) + .extracting(StatusRuntimeException::getStatus) + .isEqualTo(Status.RESOURCE_EXHAUSTED); + } + + @Test + void readAuth() { + when(backupManager.generateReadAuth(any(), eq(3))).thenReturn(Map.of("key", "value")); + + final GetCdnCredentialsResponse response = unauthenticatedServiceStub().getCdnCredentials( + GetCdnCredentialsRequest.newBuilder() + .setCdn(3) + .setSignedPresentation(signedPresentation(presentation)) + .build()); + assertThat(response.getHeadersMap()).containsExactlyEntriesOf(Map.of("key", "value")); + } + + private static AuthenticatedBackupUser backupUser(final byte[] backupId, final BackupCredentialType credentialType, + final BackupLevel backupLevel) { + return new AuthenticatedBackupUser(backupId, credentialType, backupLevel, "myBackupDir", "myMediaDir"); + } + + private static BackupAuthCredentialPresentation presentation(BackupAuthTestUtil backupAuthTestUtil, + byte[] messagesBackupKey, UUID aci) { + try { + return backupAuthTestUtil.getPresentation(BackupLevel.PAID, messagesBackupKey, aci); + } catch (VerificationFailedException e) { + throw new RuntimeException(e); + } + } + + private static SignedPresentation signedPresentation(BackupAuthCredentialPresentation presentation) { + return SignedPresentation.newBuilder() + .setPresentation(ByteString.copyFrom(presentation.serialize())) + .setPresentationSignature(ByteString.copyFromUtf8("aaa")).build(); + } + +} diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/grpc/BackupsGrpcServiceTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/grpc/BackupsGrpcServiceTest.java new file mode 100644 index 000000000..7a6e5c42e --- /dev/null +++ b/service/src/test/java/org/whispersystems/textsecuregcm/grpc/BackupsGrpcServiceTest.java @@ -0,0 +1,243 @@ +/* + * Copyright 2023 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.whispersystems.textsecuregcm.grpc; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.protobuf.ByteString; +import io.grpc.Status; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; +import org.junitpioneer.jupiter.cartesian.CartesianTest; +import org.mockito.Mock; +import org.signal.chat.backup.BackupsGrpc; +import org.signal.chat.backup.GetBackupAuthCredentialsRequest; +import org.signal.chat.backup.GetBackupAuthCredentialsResponse; +import org.signal.chat.backup.RedeemReceiptRequest; +import org.signal.chat.backup.SetBackupIdRequest; +import org.signal.chat.common.ZkCredential; +import org.signal.libsignal.zkgroup.InvalidInputException; +import org.signal.libsignal.zkgroup.ServerSecretParams; +import org.signal.libsignal.zkgroup.VerificationFailedException; +import org.signal.libsignal.zkgroup.backups.BackupAuthCredentialRequest; +import org.signal.libsignal.zkgroup.backups.BackupCredentialType; +import org.signal.libsignal.zkgroup.backups.BackupLevel; +import org.signal.libsignal.zkgroup.receipts.ClientZkReceiptOperations; +import org.signal.libsignal.zkgroup.receipts.ReceiptCredential; +import org.signal.libsignal.zkgroup.receipts.ReceiptCredentialPresentation; +import org.signal.libsignal.zkgroup.receipts.ReceiptCredentialRequestContext; +import org.signal.libsignal.zkgroup.receipts.ReceiptCredentialResponse; +import org.signal.libsignal.zkgroup.receipts.ReceiptSerial; +import org.signal.libsignal.zkgroup.receipts.ServerZkReceiptOperations; +import org.whispersystems.textsecuregcm.backup.BackupAuthManager; +import org.whispersystems.textsecuregcm.backup.BackupAuthTestUtil; +import org.whispersystems.textsecuregcm.controllers.RateLimitExceededException; +import org.whispersystems.textsecuregcm.storage.Account; +import org.whispersystems.textsecuregcm.storage.AccountsManager; +import org.whispersystems.textsecuregcm.util.EnumMapUtil; +import org.whispersystems.textsecuregcm.util.TestRandomUtil; + +class BackupsGrpcServiceTest extends SimpleBaseGrpcTest { + + private final byte[] messagesBackupKey = TestRandomUtil.nextBytes(32); + private final byte[] mediaBackupKey = TestRandomUtil.nextBytes(32); + private final BackupAuthTestUtil backupAuthTestUtil = new BackupAuthTestUtil(Clock.systemUTC()); + final BackupAuthCredentialRequest mediaAuthCredRequest = + backupAuthTestUtil.getRequest(mediaBackupKey, AUTHENTICATED_ACI); + final BackupAuthCredentialRequest messagesAuthCredRequest = + backupAuthTestUtil.getRequest(messagesBackupKey, AUTHENTICATED_ACI); + private final Account account = mock(Account.class); + + @Mock + private BackupAuthManager backupAuthManager; + @Mock + private AccountsManager accountsManager; + + @Override + protected BackupsGrpcService createServiceBeforeEachTest() { + return new BackupsGrpcService(accountsManager, backupAuthManager); + } + + @BeforeEach + void setup() { + when(accountsManager.getByAccountIdentifierAsync(AUTHENTICATED_ACI)) + .thenReturn(CompletableFuture.completedFuture(Optional.of(account))); + } + + + @Test + void setBackupId() { + when(backupAuthManager.commitBackupId(any(), any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + + authenticatedServiceStub().setBackupId( + SetBackupIdRequest.newBuilder() + .setMediaBackupAuthCredentialRequest(ByteString.copyFrom(mediaAuthCredRequest.serialize())) + .setMessagesBackupAuthCredentialRequest(ByteString.copyFrom(messagesAuthCredRequest.serialize())) + .build()); + + verify(backupAuthManager).commitBackupId(account, messagesAuthCredRequest, mediaAuthCredRequest); + } + + @Test + void setBackupIdInvalid() { + // missing media credential + GrpcTestUtils.assertStatusException( + Status.INVALID_ARGUMENT, () -> authenticatedServiceStub().setBackupId(SetBackupIdRequest.newBuilder() + .setMessagesBackupAuthCredentialRequest(ByteString.copyFrom(messagesAuthCredRequest.serialize())) + .build()) + ); + + // missing message credential + GrpcTestUtils.assertStatusException( + Status.INVALID_ARGUMENT, () -> authenticatedServiceStub().setBackupId(SetBackupIdRequest.newBuilder() + .setMediaBackupAuthCredentialRequest(ByteString.copyFrom(mediaAuthCredRequest.serialize())) + .build()) + ); + + // missing all credentials + GrpcTestUtils.assertStatusException( + Status.INVALID_ARGUMENT, () -> authenticatedServiceStub().setBackupId(SetBackupIdRequest.newBuilder().build()) + ); + + // invalid serialization + GrpcTestUtils.assertStatusException( + Status.INVALID_ARGUMENT, () -> authenticatedServiceStub().setBackupId( + SetBackupIdRequest.newBuilder() + .setMessagesBackupAuthCredentialRequest(ByteString.fromHex("FF")) + .setMediaBackupAuthCredentialRequest(ByteString.fromHex("FF")) + .build()) + ); + + } + + public static Stream setBackupIdException() { + return Stream.of( + Arguments.of(new RateLimitExceededException(null), false, Status.RESOURCE_EXHAUSTED), + Arguments.of(Status.INVALID_ARGUMENT.withDescription("async").asRuntimeException(), false, + Status.INVALID_ARGUMENT), + Arguments.of(Status.INVALID_ARGUMENT.withDescription("sync").asRuntimeException(), true, + Status.INVALID_ARGUMENT) + ); + } + + @ParameterizedTest + @MethodSource + void setBackupIdException(final Exception ex, final boolean sync, final Status expected) { + if (sync) { + when(backupAuthManager.commitBackupId(any(), any(), any())).thenThrow(ex); + } else { + when(backupAuthManager.commitBackupId(any(), any(), any())).thenReturn(CompletableFuture.failedFuture(ex)); + } + + GrpcTestUtils.assertStatusException( + expected, () -> authenticatedServiceStub().setBackupId(SetBackupIdRequest.newBuilder() + .setMediaBackupAuthCredentialRequest(ByteString.copyFrom(mediaAuthCredRequest.serialize())) + .setMessagesBackupAuthCredentialRequest(ByteString.copyFrom(messagesAuthCredRequest.serialize())) + .build()) + ); + } + + @Test + void redeemReceipt() throws InvalidInputException, VerificationFailedException { + final ServerSecretParams params = ServerSecretParams.generate(); + final ServerZkReceiptOperations serverOps = new ServerZkReceiptOperations(params); + final ClientZkReceiptOperations clientOps = new ClientZkReceiptOperations(params.getPublicParams()); + final ReceiptCredentialRequestContext rcrc = clientOps + .createReceiptCredentialRequestContext(new ReceiptSerial(TestRandomUtil.nextBytes(ReceiptSerial.SIZE))); + final ReceiptCredentialResponse rcr = serverOps.issueReceiptCredential(rcrc.getRequest(), 0L, 3L); + final ReceiptCredential receiptCredential = clientOps.receiveReceiptCredential(rcrc, rcr); + final ReceiptCredentialPresentation presentation = clientOps.createReceiptCredentialPresentation(receiptCredential); + + when(backupAuthManager.redeemReceipt(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + + authenticatedServiceStub().redeemReceipt(RedeemReceiptRequest.newBuilder() + .setPresentation(ByteString.copyFrom(presentation.serialize())) + .build()); + + verify(backupAuthManager).redeemReceipt(account, presentation); + } + + + @Test + void getCredentials() { + final Instant start = Instant.now().truncatedTo(ChronoUnit.DAYS); + final Instant end = start.plus(Duration.ofDays(1)); + + final Map> expectedCredentialsByType = + EnumMapUtil.toEnumMap(BackupCredentialType.class, credentialType -> backupAuthTestUtil.getCredentials( + BackupLevel.PAID, backupAuthTestUtil.getRequest(messagesBackupKey, AUTHENTICATED_ACI), credentialType, + start, end)); + + expectedCredentialsByType.forEach((credentialType, expectedCredentials) -> + when(backupAuthManager.getBackupAuthCredentials(any(), eq(credentialType), eq(start), eq(end))) + .thenReturn(CompletableFuture.completedFuture(expectedCredentials))); + + final GetBackupAuthCredentialsResponse credentialResponse = authenticatedServiceStub().getBackupAuthCredentials( + GetBackupAuthCredentialsRequest.newBuilder() + .setRedemptionStart(start.getEpochSecond()).setRedemptionStop(end.getEpochSecond()) + .build()); + + expectedCredentialsByType.forEach((credentialType, expectedCredentials) -> { + + final Map creds = switch (credentialType) { + case MESSAGES -> credentialResponse.getMessageCredentialsMap(); + case MEDIA -> credentialResponse.getMediaCredentialsMap(); + }; + assertThat(creds).hasSize(expectedCredentials.size()).containsKey(start.getEpochSecond()); + + for (BackupAuthManager.Credential expectedCred : expectedCredentials) { + assertThat(creds) + .extractingByKey(expectedCred.redemptionTime().getEpochSecond()) + .isNotNull() + .extracting(ZkCredential::getCredential) + .extracting(ByteString::toByteArray) + .isEqualTo(expectedCred.credential().serialize()); + } + }); + } + + @ParameterizedTest + @CsvSource({ + "true, false", + "false, true", + "true, true" + }) + void getCredentialsBadInput(final boolean missingStart, final boolean missingEnd) { + final Instant start = Instant.now().truncatedTo(ChronoUnit.DAYS); + final Instant end = start.plus(Duration.ofDays(1)); + + final GetBackupAuthCredentialsRequest.Builder builder = GetBackupAuthCredentialsRequest.newBuilder(); + if (!missingStart) { + builder.setRedemptionStart(start.getEpochSecond()); + } + if (!missingEnd) { + builder.setRedemptionStop(end.getEpochSecond()); + } + + GrpcTestUtils.assertStatusException(Status.INVALID_ARGUMENT, + () -> authenticatedServiceStub().getBackupAuthCredentials(builder.build())); + } + +} diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/grpc/GrpcTestUtils.java b/service/src/test/java/org/whispersystems/textsecuregcm/grpc/GrpcTestUtils.java index a2724b1bd..65c3c97a3 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/grpc/GrpcTestUtils.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/grpc/GrpcTestUtils.java @@ -35,7 +35,7 @@ public static void setupAuthenticatedExtension( final BindableService service) { mockAuthenticationInterceptor.setAuthenticatedDevice(authenticatedAci, authenticatedDeviceId); extension.getServiceRegistry() - .addService(ServerInterceptors.intercept(service, mockRequestAttributesInterceptor, mockAuthenticationInterceptor, new ErrorMappingInterceptor())); + .addService(ServerInterceptors.intercept(service, new ValidatingInterceptor(), mockRequestAttributesInterceptor, mockAuthenticationInterceptor, new ErrorMappingInterceptor())); } public static void setupUnauthenticatedExtension( @@ -43,7 +43,7 @@ public static void setupUnauthenticatedExtension( final MockRequestAttributesInterceptor mockRequestAttributesInterceptor, final BindableService service) { extension.getServiceRegistry() - .addService(ServerInterceptors.intercept(service, mockRequestAttributesInterceptor, new ErrorMappingInterceptor())); + .addService(ServerInterceptors.intercept(service, new ValidatingInterceptor(), mockRequestAttributesInterceptor, new ErrorMappingInterceptor())); } public static void assertStatusException(final Status expected, final Executable serviceCall) {