Merge pull request #6153 from decentraland/feat/sdk7-audio-stream-aud…

…io-event-component

feat: sdk7 audio stream audio event component

browser-interface/package.json

@@ -56,14 +56,14 @@
   },
   "dependencies": {
     "@dcl/crypto": "^3.3.1",
-    "@dcl/ecs": "7.3.40",
+    "@dcl/ecs": "7.4.16",
     "@dcl/ecs-math": "^1.0.3",
     "@dcl/ecs-quests": "^1.3.1",
     "@dcl/feature-flags": "^1.1.0",
     "@dcl/hashing": "^1.1.3",
     "@dcl/kernel-interface": "^2.0.0-20230512115658.commit-b582e05",
     "@dcl/legacy-ecs": "^6.11.11",
-    "@dcl/protocol": "1.0.0-7716486147.commit-7433b10",
+    "@dcl/protocol": "1.0.0-8691799990.commit-4ba546c",
     "@dcl/rpc": "^1.1.1",
     "@dcl/scene-runtime": "7.0.6-20240220184109.commit-cf1e4e2",
     "@dcl/schemas": "^9.1.1",

browser-interface/packages/shared/apis/host/EngineAPI.ts

@@ -69,7 +69,6 @@ export function registerEngineApiServiceServerImplementation(port: RpcServerPort
           // PlayerComponents messages
           const internalAvatarMessages = (await ctx.internalEngine?.update()) ?? []
 
-
           return {
             hasEntities: response.hasOwnEntities || hasMainCrdt,
             // send the initialEntitiesTick0 (main.crdt) and the response.payload

browser-interface/packages/shared/world/SceneWorker.ts

@@ -429,7 +429,7 @@ export class SceneWorker {
     })
     if (this.rpcContext.sdk7) {
       this.rpcContext.internalEngine = createInternalEngine(
-        this.rpcContext.sceneData.id,
+        this.rpcContext.sceneData.sceneNumber,
         this.metadata.scene.parcels,
         showAsPortableExperience
       )

browser-interface/packages/shared/world/runtime-7/engine.ts

@@ -1,5 +1,7 @@
 import { Engine, IEngine, Transport } from '@dcl/ecs/dist-cjs'
 import {
+  MediaState,
+  AudioEvent as defineAudioEvent,
   Transform as defineTransform,
   PlayerIdentityData as definePlayerIdentityData,
   AvatarBase as defineAvatarBase,
@@ -37,6 +39,16 @@ type LocalProfileChange = {
   triggerEmote: EmoteData
 }
 
+type State = {
+  sceneId: number
+  entityId: Entity
+  state: MediaState
+}
+
+type AudioStreamChange = {
+  changeState: State
+}
+
 function getUserData(userId: string) {
   const dataFromStore = getProfileFromStore(store.getState(), userId)
   if (!dataFromStore) return undefined
@@ -48,12 +60,13 @@ function getUserData(userId: string) {
 }
 
 export const localProfileChanged = mitt<LocalProfileChange>()
+export const audioStreamEmitter = mitt<AudioStreamChange>()
 
 /**
  * We used this engine as an internal engine to add information to the worker.
  * It handles the Avatar information for each player
  */
-export function createInternalEngine(id: string, parcels: string[], isGlobalScene: boolean): IInternalEngine {
+export function createInternalEngine(sceneNumber: number, parcels: string[], isGlobalScene: boolean): IInternalEngine {
   const AVATAR_RESERVED_ENTITIES = { from: 10, to: 200 }
   const userId = getCurrentUserId(store.getState())!
 
@@ -65,6 +78,7 @@ export function createInternalEngine(id: string, parcels: string[], isGlobalScen
   const AvatarEquippedData = defineAvatarEquippedData(engine)
   const PlayerIdentityData = definePlayerIdentityData(engine)
   const AvatarEmoteCommand = defineAvatarEmoteCommand(engine)
+  const AudioEvent = defineAudioEvent(engine)
   const avatarMap = new Map<string, Entity>()
 
   function addUser(userId: string) {
@@ -191,6 +205,16 @@ export function createInternalEngine(id: string, parcels: string[], isGlobalScen
   })
   // End of LOCAL USER
 
+  // AudioStream updates
+  audioStreamEmitter.on('changeState', ({ entityId, sceneId, state }) => {
+    if (sceneId !== sceneNumber) return
+    const audioEvent = AudioEvent.get(entityId)
+    const lastAudio = [...audioEvent.values()].pop()
+    const timestamp = (lastAudio?.timestamp ?? 0) + 1
+    AudioEvent.addValue(entityId, { state, timestamp })
+  })
+  // end of AudioStream updates
+
   // PlayersConnected observers
   const observerInstance = avatarMessageObservable.add((message) => {
     if (message.type === AvatarMessageType.USER_EXPRESSION) {
@@ -219,18 +243,18 @@ export function createInternalEngine(id: string, parcels: string[], isGlobalScen
   })
 
   /**
-   * We used this transport to send only the avatar updates to the client instead of the full state
-   * Every time there is an update on a profile, this would add those CRDT messages to avatarMessages
+   * We used this transport to send only kernel-side updates (profile, emotes, audio stream...) to the client instead of the full state
+   * For example: every time there is an update on a profile, this would add those CRDT messages to internalMessages
    * and they'd be append to the crdtSendToRenderer call
    */
   const transport: Transport = {
     filter: (message) => !!message,
     send: async (message: Uint8Array) => {
-      if (message.byteLength) avatarMessages.push(message)
+      if (message.byteLength) internalMessages.push(message)
     }
   }
   engine.addTransport(transport)
-  const avatarMessages: Uint8Array[] = []
+  const internalMessages: Uint8Array[] = []
 
   // Add current user
   addUser(userId)
@@ -246,8 +270,8 @@ export function createInternalEngine(id: string, parcels: string[], isGlobalScen
     engine,
     update: async () => {
       await engine.update(1)
-      const messages = [...avatarMessages]
-      avatarMessages.length = 0
+      const messages = [...internalMessages]
+      internalMessages.length = 0
       return messages
     },
     destroy: () => {

browser-interface/packages/unity-interface/BrowserInterface.ts

@@ -26,7 +26,7 @@ import { notifyStatusThroughChat } from 'shared/chat'
 import { sendMessage } from 'shared/chat/actions'
 import { sendPublicChatMessage } from 'shared/comms'
 import { changeRealm } from 'shared/dao'
-import {getExploreRealmsService, getSelectedNetwork} from 'shared/dao/selectors'
+import { getExploreRealmsService, getSelectedNetwork } from 'shared/dao/selectors'
 import { getERC20Balance } from 'lib/web3/EthereumService'
 import { leaveChannel, updateUserData } from 'shared/friends/actions'
 import { ensureFriendProfile } from 'shared/friends/ensureFriendProfile'
@@ -113,13 +113,14 @@ import {
 } from 'shared/world/parcelSceneManager'
 import { receivePositionReport } from 'shared/world/positionThings'
 import { TeleportController } from 'shared/world/TeleportController'
-import { setAudioStream } from './audioStream'
+import { setAudioStream, killAudioStream, setAudioStreamForEntity } from './audioStream'
 import { fetchENSOwnerProfile } from './fetchENSOwnerProfile'
 import { GIFProcessor } from './gif-processor'
 import { getUnityInstance } from './IUnityInterface'
 import { encodeParcelPosition } from 'lib/decentraland'
 import { Vector2 } from 'shared/protocol/decentraland/common/vectors.gen'
 import {fetchAndReportRealmsInfo} from "../shared/renderer/sagas";
+import { Entity } from '@dcl/ecs/dist-cjs'
 
 declare const globalThis: { gifProcessor?: GIFProcessor; __debug_wearables: any }
 export const futures: Record<string, IFuture<any>> = {}
@@ -849,6 +850,14 @@ export class BrowserInterface {
     setAudioStream(data.url, data.play, data.volume).catch((err) => defaultLogger.log(err))
   }
 
+  public SetAudioStreamForEntity(data: { url: string; play: boolean; volume: number; sceneNumber: number; entityId: Entity }) {
+    setAudioStreamForEntity(data.url, data.play, data.volume, data.sceneNumber, data.entityId).catch((err) => defaultLogger.log(err))
+  }
+
+  public KillAudioStream(data: { sceneNumber: number; entityId: Entity }) {
+    killAudioStream(data.sceneNumber, data.entityId).catch((err) => defaultLogger.log(err))
+  }
+
   public SendChatMessage(data: { message: ChatMessage }) {
     store.dispatch(sendMessage(data.message))
   }

browser-interface/packages/unity-interface/audioStream.ts

@@ -1,7 +1,70 @@
 /////////////////////////////////// AUDIO STREAMING ///////////////////////////////////
+import { Entity } from '@dcl/ecs/dist-cjs'
+import { MediaState } from '@dcl/ecs/dist-cjs/components'
+
+import { audioStreamEmitter } from '../shared/world/runtime-7/engine'
+
+type AudioEvents = HTMLMediaElementEventMap
+type GlobalProps = {
+  playToken: number
+  entityId?: Entity
+  sceneId?: number
+}
+
+let globalProps: GlobalProps = {
+  playToken: 0,
+  sceneId: undefined,
+  entityId: undefined
+}
+
+function setGlobalProps(props: GlobalProps) {
+  globalProps = props
+}
+
+// https://developer.mozilla.org/en-US/docs/Web/API/HTMLMediaElement#events
+const AUDIO_EVENTS: (keyof AudioEvents)[] = [
+  'loadeddata',
+  'error',
+  'seeking',
+  'loadstart',
+  'waiting',
+  'playing',
+  'pause'
+]
 
 const audioStreamSource = new Audio()
-let playToken: number = 0;
+AUDIO_EVENTS.forEach(($) => audioStreamSource.addEventListener($, listen))
+
+function listen(ev: AudioEvents[keyof AudioEvents]) {
+  const { entityId, sceneId } = globalProps
+
+  if (!entityId || !sceneId) return
+
+  const state = getAudioEvent(ev.type)
+
+  audioStreamEmitter.emit('changeState', { entityId, state, sceneId })
+}
+
+function getAudioEvent(type: string): MediaState {
+  switch (type) {
+    case 'loadeddata':
+      return MediaState.MS_READY
+    case 'error':
+      return MediaState.MS_ERROR
+    case 'seeking':
+      return MediaState.MS_SEEKING
+    case 'loadstart':
+      return MediaState.MS_LOADING
+    case 'waiting':
+      return MediaState.MS_BUFFERING
+    case 'playing':
+      return MediaState.MS_PLAYING
+    case 'pause':
+      return MediaState.MS_PAUSED
+    default:
+      return MediaState.MS_NONE
+  }
+}
 
 export async function setAudioStream(url: string, play: boolean, volume: number) {
   const isSameSrc =
@@ -13,7 +76,7 @@ export async function setAudioStream(url: string, play: boolean, volume: number)
   if (play && !isSameSrc) {
     audioStreamSource.src = url
   } else if (!play && isSameSrc) {
-    playToken++
+    globalProps.playToken++
     audioStreamSource.pause()
   }
 
@@ -22,19 +85,34 @@ export async function setAudioStream(url: string, play: boolean, volume: number)
   }
 }
 
+export async function setAudioStreamForEntity(
+  url: string,
+  play: boolean,
+  volume: number,
+  sceneId: number,
+  entityId: Entity
+) {
+  setGlobalProps({ ...globalProps, sceneId, entityId })
+  void setAudioStream(url, play, volume)
+}
+
+export async function killAudioStream(sceneId: number, entityId: Entity) {
+  setGlobalProps({ ...globalProps, sceneId: undefined, entityId: undefined })
+  void setAudioStream(audioStreamSource.src, false, audioStreamSource.volume)
+  audioStreamEmitter.emit('changeState', { entityId, state: MediaState.MS_NONE, sceneId })
+}
+
 // audioStreamSource play might be requested without user interaction
 // i.e: spawning in world without clicking on the canvas
 // so me want to keep retrying on play exception until audio starts playing
 function playIntent() {
   function tryPlay(token: number) {
-    if (playToken !== token)
-      return
+    if (globalProps.playToken !== token) return
 
-    audioStreamSource.play()
-      .catch(_ => {
-        setTimeout(() => tryPlay(token), 500)
-      })
+    audioStreamSource.play().catch((_) => {
+      setTimeout(() => tryPlay(token), 500)
+    })
   }
-  playToken++
-  tryPlay(playToken)
+  globalProps.playToken++
+  tryPlay(globalProps.playToken)
 }

unity-renderer/Assets/DCLPlugins/ECS7/ECSComponents/AudioStream/ECSAudioStreamComponentHandler.cs

@@ -18,6 +18,7 @@ public class ECSAudioStreamComponentHandler : IECSComponentHandler<PBAudioStream
         internal bool isPlaying = false;
         internal PBAudioStream model;
         internal IParcelScene scene;
+        internal IDCLEntity entity;
         internal string url;
 
         // Flags to check if we can activate the AudioStream
@@ -28,6 +29,7 @@ public class ECSAudioStreamComponentHandler : IECSComponentHandler<PBAudioStream
         public void OnComponentCreated(IParcelScene scene, IDCLEntity entity)
         {
             this.scene = scene;
+            this.entity = entity;
 
             if (!scene.isPersistent)
                 CommonScriptableObjects.sceneNumber.OnChange += OnSceneChanged;
@@ -48,7 +50,7 @@ public void OnComponentRemoved(IParcelScene scene, IDCLEntity entity)
             Settings.i.audioSettings.OnChanged -= OnSettingsChanged;
             DataStore.i.virtualAudioMixer.sceneSFXVolume.OnChange -= SceneSFXVolume_OnChange;
 
-            StopStreaming();
+            WebInterface.KillAudioStream(scene.sceneData.sceneNumber, entity.entityId);
         }
 
         public void OnComponentModelUpdated(IParcelScene scene, IDCLEntity entity, PBAudioStream model)
@@ -162,7 +164,7 @@ private void StartStreaming()
         private void SendUpdateAudioStreamEvent(bool play)
         {
             isPlaying = play;
-            WebInterface.SendAudioStreamEvent(url, isPlaying, currentVolume);
+            WebInterface.SendAudioStreamEventForEntity(url, isPlaying, currentVolume, scene.sceneData.sceneNumber, entity.entityId);
         }
     }
 }