Part 2 - Encryption: Add version to EncryptedData and switch to binar…

…y channel payload encoding (#2097)

this ended up being a large change - but mostly to just get the
persistence layer changed over.

Olm still encrypts and decrypts strings…

For new encrypted data code where `version > 0`
- accept bytes to encode, use bin_toHex to convert to string before
encrypting
- after decrypting, convert back to bytes with bin_fromHex before
returning

For old data
- after decrypting, return string

packages/encryption/src/base.ts

@@ -46,7 +46,8 @@ export abstract class EncryptionAlgorithm implements IEncryptionParams {
         opts?: { awaitInitialShareSession: boolean },
     ): Promise<void>
 
-    abstract encrypt(streamId: string, payload: string): Promise<EncryptedData>
+    abstract encrypt_deprecated_v0(streamId: string, payload: string): Promise<EncryptedData>
+    abstract encrypt(streamId: string, payload: Uint8Array): Promise<EncryptedData>
 }
 
 /**

packages/encryption/src/decryptionExtensions.ts

@@ -713,6 +713,7 @@ export abstract class BaseDecryptionExtensions {
             return
         }
 
+        // todo split this up by algorithm so that we can send all the new hybrid keys
         knownSessionIds.sort()
         const requestedSessionIds = new Set(item.solicitation.sessionIds.sort())
         const replySessionIds = item.solicitation.isNewDevice

packages/encryption/src/groupEncryption.ts

@@ -1,7 +1,7 @@
-import { EncryptedData } from '@river-build/proto'
+import { EncryptedData, EncryptedDataVersion } from '@river-build/proto'
 import { EncryptionAlgorithm, IEncryptionParams } from './base'
 import { GroupEncryptionAlgorithmId } from './olmLib'
-import { dlog } from '@river-build/dlog'
+import { bin_toBase64, dlog } from '@river-build/dlog'
 
 const log = dlog('csb:encryption:groupEncryption')
 
@@ -71,23 +71,36 @@ export class GroupEncryption extends EncryptionAlgorithm {
         )
     }
 
+    /**
+     * @deprecated
+     */
+    public async encrypt_deprecated_v0(streamId: string, payload: string): Promise<EncryptedData> {
+        await this.ensureOutboundSession(streamId)
+        const result = await this.device.encryptGroupMessage(payload, streamId)
+        return new EncryptedData({
+            algorithm: this.algorithm,
+            senderKey: this.device.deviceCurve25519Key!,
+            ciphertext: result.ciphertext,
+            sessionId: result.sessionId,
+            version: EncryptedDataVersion.ENCRYPTED_DATA_VERSION_0,
+        })
+    }
+
     /**
      * @param content - plaintext event content
      *
      * @returns Promise which resolves to the new event body
      */
-    public async encrypt(streamId: string, payload: string): Promise<EncryptedData> {
+    public async encrypt(streamId: string, payload: Uint8Array): Promise<EncryptedData> {
         log('Starting to encrypt event')
-
         await this.ensureOutboundSession(streamId)
-
-        const result = await this.device.encryptGroupMessage(payload, streamId)
-
+        const result = await this.device.encryptGroupMessage(bin_toBase64(payload), streamId)
         return new EncryptedData({
             algorithm: this.algorithm,
             senderKey: this.device.deviceCurve25519Key!,
             ciphertext: result.ciphertext,
             sessionId: result.sessionId,
+            version: EncryptedDataVersion.ENCRYPTED_DATA_VERSION_1,
         })
     }
 }

packages/encryption/src/groupEncryptionCrypto.ts

@@ -161,11 +161,24 @@ export class GroupEncryptionCrypto {
      */
     public async encryptGroupEvent(
         streamId: string,
-        payload: string,
+        payload: Uint8Array,
         algorithm: GroupEncryptionAlgorithmId,
     ): Promise<EncryptedData> {
         return this.groupEncryption[algorithm].encrypt(streamId, payload)
     }
+    /**
+     * Deprecated uses v0 encryption version
+     *
+     * @returns Promise which resolves when the event has been
+     *     encrypted, or null if nothing was needed
+     */
+    public async encryptGroupEvent_deprecated_v0(
+        streamId: string,
+        payload: string,
+        algorithm: GroupEncryptionAlgorithmId,
+    ): Promise<EncryptedData> {
+        return this.groupEncryption[algorithm].encrypt_deprecated_v0(streamId, payload)
+    }
     /**
      * Decrypt a received event using group encryption algorithm
      *

packages/encryption/src/hybridGroupEncryption.ts

@@ -1,4 +1,4 @@
-import { EncryptedData, HybridGroupSessionKey } from '@river-build/proto'
+import { EncryptedData, EncryptedDataVersion, HybridGroupSessionKey } from '@river-build/proto'
 import { EncryptionAlgorithm, IEncryptionParams } from './base'
 import { GroupEncryptionAlgorithmId } from './olmLib'
 import { dlog } from '@river-build/dlog'
@@ -89,29 +89,41 @@ export class HybridGroupEncryption extends EncryptionAlgorithm {
         )
     }
 
+    /**
+     * @deprecated
+     */
+    public async encrypt_deprecated_v0(streamId: string, payload: string): Promise<EncryptedData> {
+        const sessionKey: HybridGroupSessionKey = await this._ensureOutboundSession(streamId)
+        const key = await importAesGsmKeyBytes(sessionKey.key)
+        const payloadBytes = new TextEncoder().encode(payload)
+        const { ciphertext, iv } = await encryptAesGcm(key, payloadBytes)
+        return new EncryptedData({
+            algorithm: this.algorithm,
+            senderKey: this.device.deviceCurve25519Key!,
+            sessionIdBytes: sessionKey.sessionId,
+            ciphertextBytes: ciphertext,
+            ivBytes: iv,
+            version: EncryptedDataVersion.ENCRYPTED_DATA_VERSION_0,
+        })
+    }
+
     /**
      * @param content - plaintext event content
      *
      * @returns Promise which resolves to the new event body
      */
-    public async encrypt(streamId: string, payload: string | Uint8Array): Promise<EncryptedData> {
+    public async encrypt(streamId: string, payload: Uint8Array): Promise<EncryptedData> {
         log('Starting to encrypt event')
-
-        const payloadBytes =
-            typeof payload === 'string' ? new TextEncoder().encode(payload) : payload
-
         const sessionKey: HybridGroupSessionKey = await this._ensureOutboundSession(streamId)
-
         const key = await importAesGsmKeyBytes(sessionKey.key)
-
-        const { ciphertext, iv } = await encryptAesGcm(key, payloadBytes)
-
+        const { ciphertext, iv } = await encryptAesGcm(key, payload)
         return new EncryptedData({
             algorithm: this.algorithm,
             senderKey: this.device.deviceCurve25519Key!,
             sessionIdBytes: sessionKey.sessionId,
             ciphertextBytes: ciphertext,
             ivBytes: iv,
+            version: EncryptedDataVersion.ENCRYPTED_DATA_VERSION_1,
         })
     }
 }

packages/encryption/src/tests/decryptionExtensions.test.ts

@@ -61,6 +61,11 @@ describe.concurrent('TestDecryptionExtensions', () => {
             bobDex.start()
             // bob encrypts a message
             const encryptedData = await bobCrypto.encryptGroupEvent(
+                streamId,
+                new TextEncoder().encode(bobsPlaintext),
+                algorithm,
+            )
+            const encryptedData_V0 = await bobCrypto.encryptGroupEvent_deprecated_v0(
                 streamId,
                 bobsPlaintext,
                 algorithm,
@@ -89,17 +94,22 @@ describe.concurrent('TestDecryptionExtensions', () => {
 
             // try to decrypt the message
             const decrypted = await aliceDex.crypto.decryptGroupEvent(streamId, encryptedData)
+            const decrypted_V0 = await aliceDex.crypto.decryptGroupEvent(streamId, encryptedData_V0)
 
-            if (typeof decrypted !== 'string') {
+            if (typeof decrypted === 'string') {
                 throw new Error('decrypted is a string') // v1 should be bytes
             }
+            if (typeof decrypted_V0 !== 'string') {
+                throw new Error('decrypted_V0 is a string') // v0 should be bytes
+            }
 
             // stop the decryption extensions
             await bobDex.stop()
             await aliceDex.stop()
 
             // assert
-            expect(decrypted).toBe(bobsPlaintext)
+            expect(new TextDecoder().decode(decrypted)).toBe(bobsPlaintext)
+            expect(decrypted_V0).toBe(bobsPlaintext)
             expect(bobDex.seenStates).toContain(DecryptionStatus.respondingToKeyRequests)
             expect(aliceDex.seenStates).toContain(DecryptionStatus.processingNewGroupSessions)
         },
@@ -130,6 +140,11 @@ describe.concurrent('TestDecryptionExtensions', () => {
             bobDex.start()
             // bob encrypts a message
             const encryptedData = await bobCrypto.encryptGroupEvent(
+                streamId,
+                new TextEncoder().encode(bobsPlaintext),
+                algorithm,
+            )
+            const encryptedData_V0 = await bobCrypto.encryptGroupEvent_deprecated_v0(
                 streamId,
                 bobsPlaintext,
                 algorithm,
@@ -145,12 +160,23 @@ describe.concurrent('TestDecryptionExtensions', () => {
             // try to decrypt the message
             const decrypted = await aliceDex.crypto.decryptGroupEvent(streamId, encryptedData)
 
+            if (typeof decrypted === 'string') {
+                throw new Error('decrypted is a string') // v1 should be bytes
+            }
+
+            const decrypted_V0 = await aliceDex.crypto.decryptGroupEvent(streamId, encryptedData_V0)
+
+            if (typeof decrypted_V0 !== 'string') {
+                throw new Error('decrypted_V0 is a string') // v0 should be bytes
+            }
+
             // stop the decryption extensions
             await bobDex.stop()
             await aliceDex.stop()
 
             // assert
-            expect(decrypted).toBe(bobsPlaintext)
+            expect(new TextDecoder().decode(decrypted)).toBe(bobsPlaintext)
+            expect(decrypted_V0).toBe(bobsPlaintext)
         },
     )
 })

packages/sdk/src/client.ts

@@ -32,6 +32,7 @@ import {
     MiniblockHeader,
     GetStreamResponse,
     CreateStreamResponse,
+    ChannelProperties,
 } from '@river-build/proto'
 import {
     bin_fromHexString,
@@ -97,7 +98,6 @@ import { IStreamStateView, StreamStateView } from './streamStateView'
 import {
     make_UserMetadataPayload_Inception,
     make_ChannelPayload_Inception,
-    make_ChannelProperties,
     make_ChannelPayload_Message,
     make_MemberPayload_Membership2,
     make_SpacePayload_Inception,
@@ -955,10 +955,10 @@ export class Client
         assert(isGDMChannelStreamId(streamId), 'streamId must be a valid GDM stream id')
         check(isDefined(this.cryptoBackend))
 
-        const channelProps = make_ChannelProperties(channelName, channelTopic).toJsonString()
+        const channelProps = new ChannelProperties({ name: channelName, topic: channelTopic })
         const encryptedData = await this.cryptoBackend.encryptGroupEvent(
             streamId,
-            channelProps,
+            channelProps.toBinary(),
             this.defaultGroupEncryptionAlgorithm,
         )
 
@@ -1140,7 +1140,7 @@ export class Client
         check(isDefined(this.cryptoBackend))
         const encryptedData = await this.cryptoBackend.encryptGroupEvent(
             streamId,
-            displayName,
+            new TextEncoder().encode(displayName),
             this.defaultGroupEncryptionAlgorithm,
         )
         await this.makeEventAndAddToStream(
@@ -1157,7 +1157,7 @@ export class Client
         stream.view.getMemberMetadata().usernames.setLocalUsername(this.userId, username)
         const encryptedData = await this.cryptoBackend.encryptGroupEvent(
             streamId,
-            username,
+            new TextEncoder().encode(username),
             this.defaultGroupEncryptionAlgorithm,
         )
         encryptedData.checksum = usernameChecksum(username, streamId)
@@ -1614,7 +1614,7 @@ export class Client
         }
 
         const tags = opts?.disableTags === true ? undefined : makeTags(payload, stream.view)
-        const cleartext = payload.toJsonString()
+        const cleartext = payload.toBinary()
 
         let message: EncryptedData
         const encryptionAlgorithm = stream.view.membershipContent.encryptionAlgorithm
@@ -2333,7 +2333,7 @@ export class Client
         options: {
             method?: string
             localId?: string
-            cleartext?: string
+            cleartext?: Uint8Array
             optional?: boolean
             tags?: PlainMessage<Tags>
         } = {},
@@ -2375,7 +2375,7 @@ export class Client
         prevMiniblockHash: Uint8Array,
         optional?: boolean,
         localId?: string,
-        cleartext?: string,
+        cleartext?: Uint8Array,
         tags?: PlainMessage<Tags>,
         retryCount?: number,
     ): Promise<{ prevMiniblockHash: Uint8Array; eventId: string; error?: AddEventResponse_Error }> {
@@ -2665,8 +2665,8 @@ export class Client
         if (!this.cryptoBackend) {
             throw new Error('crypto backend not initialized')
         }
-        const cleartext = event.toJsonString()
-        return this.cryptoBackend.encryptGroupEvent(streamId, cleartext, algorithm)
+
+        return this.cryptoBackend.encryptGroupEvent(streamId, event.toBinary(), algorithm)
     }
 
     async encryptWithDeviceKeys(

packages/sdk/src/mls/coordinator/coordinator.ts

@@ -40,17 +40,6 @@ type MlsEncryptedContentItem = {
     encryptedData: EncryptedData
 }
 
-const encoder = new TextEncoder()
-const decoder = new TextDecoder()
-
-function encode(text: string): Uint8Array {
-    return encoder.encode(text)
-}
-
-function decode(bytes: Uint8Array): string {
-    return decoder.decode(bytes)
-}
-
 export interface ICoordinator {
     // Commands
     joinOrCreateGroup(streamId: string): Promise<void>
@@ -164,10 +153,9 @@ export class Coordinator implements ICoordinator {
             }
         }
 
-        const plaintext_ = event.toJsonString()
-        const plaintext = encode(plaintext_)
+        const plainbytes = event.toBinary()
 
-        return this.epochSecretService.encryptMessage(epochSecret, plaintext)
+        return this.epochSecretService.encryptMessage(epochSecret, plainbytes)
     }
 
     // TODO: Maybe this could be refactored into a separate class
@@ -187,11 +175,7 @@ export class Coordinator implements ICoordinator {
         // check cache
         let cleartext = await this.persistenceStore.getCleartext(eventId)
         if (cleartext === undefined) {
-            const cleartext_ = await this.epochSecretService.decryptMessage(
-                epochSecret,
-                encryptedData,
-            )
-            cleartext = decode(cleartext_)
+            cleartext = await this.epochSecretService.decryptMessage(epochSecret, encryptedData)
         }
         const decryptedContent = toDecryptedContent(kind, encryptedData.version, cleartext)
 

packages/sdk/src/mls/epoch/epochSecretService.ts

@@ -7,7 +7,11 @@ import {
 } from '@river-build/mls-rs-wasm'
 import { bin_toHexString, dlog, DLogger, shortenHexString } from '@river-build/dlog'
 import { DerivedKeys, EpochSecret, EpochSecretId, epochSecretId } from './epochSecret'
-import { EncryptedData, MemberPayload_Mls_EpochSecrets } from '@river-build/proto'
+import {
+    EncryptedData,
+    EncryptedDataVersion,
+    MemberPayload_Mls_EpochSecrets,
+} from '@river-build/proto'
 import { IEpochSecretStore } from './epochSecretStore'
 import { PlainMessage } from '@bufbuild/protobuf'
 import { MLS_ALGORITHM } from '../constants'
@@ -213,6 +217,7 @@ export class EpochSecretService {
                 epoch: epochSecret.epoch,
                 ciphertext,
             },
+            version: EncryptedDataVersion.ENCRYPTED_DATA_VERSION_1,
         })
     }
 

packages/sdk/src/types.ts

@@ -6,7 +6,6 @@ import {
     UserMetadataPayload_Inception,
     UserPayload_Inception,
     SpacePayload_Inception,
-    ChannelProperties,
     ChannelPayload_Inception,
     UserSettingsPayload_Inception,
     SpacePayload_ChannelUpdate,
@@ -458,13 +457,6 @@ export const make_ChannelMessage_Redaction = (
     })
 }
 
-export const make_ChannelProperties = (
-    channelName: string,
-    channelTopic: string,
-): ChannelProperties => {
-    return new ChannelProperties({ name: channelName, topic: channelTopic })
-}
-
 export const make_ChannelPayload_Inception = (
     value: PlainMessage<ChannelPayload_Inception>,
 ): PlainMessage<StreamEvent>['payload'] => {