Opus Interceptor

src/examples/java/AudioEchoExample.java

@@ -19,7 +19,6 @@
 import net.dv8tion.jda.api.audio.AudioReceiveHandler;
 import net.dv8tion.jda.api.audio.AudioSendHandler;
 import net.dv8tion.jda.api.audio.CombinedAudio;
-import net.dv8tion.jda.api.audio.UserAudio;
 import net.dv8tion.jda.api.entities.*;
 import net.dv8tion.jda.api.events.message.guild.GuildMessageReceivedEvent;
 import net.dv8tion.jda.api.hooks.ListenerAdapter;
@@ -202,13 +201,6 @@ public boolean canReceiveCombined()
             return queue.size() < 10;
         }
 
-        @Override // give audio separately for each user that is speaking
-        public boolean canReceiveUser()
-        {
-            // this is not useful if we want to echo the audio of the voice channel, thus disabled for this purpose
-            return false;
-        }
-
         @Override
         public void handleCombinedAudio(CombinedAudio combinedAudio)
         {
@@ -219,9 +211,19 @@ public void handleCombinedAudio(CombinedAudio combinedAudio)
             byte[] data = combinedAudio.getAudioData(1.0f); // volume at 100% = 1.0 (50% = 0.5 / 55% = 0.55)
             queue.add(data);
         }
+/*
+        Disable per-user audio since we want to echo the entire channel and not specific users.
+
+        @Override // give audio separately for each user that is speaking
+        public boolean canReceiveUser()
+        {
+            // this is not useful if we want to echo the audio of the voice channel, thus disabled for this purpose
+            return false;
+        }
 
         @Override
         public void handleUserAudio(UserAudio userAudio) {} // per-user is not helpful in an echo system
+*/
 
         /* Send Handling */
 

src/main/java/net/dv8tion/jda/api/audio/AudioReceiveHandler.java

@@ -37,14 +37,51 @@ public interface AudioReceiveHandler
      *
      * @return If true, JDA enables subsystems to combine all user audio into a single provided data packet.
      */
-    boolean canReceiveCombined();
+    default boolean canReceiveCombined()
+    {
+        return false;
+    }
 
     /**
      * If this method returns true, then JDA will provide audio data to the {@link #handleUserAudio(UserAudio)} method.
      *
      * @return If true, JDA enables subsystems to provide user specific audio data.
      */
-    boolean canReceiveUser();
+    default boolean canReceiveUser()
+    {
+        return false;
+    }
+
+    /**
+     * If this method returns true, then JDA will provide raw OPUS encoded packets to {@link #handleEncodedAudio(OpusPacket)}.
+     * <br>This can be used in combination with the other receive methods but will not be combined audio of multiple users.
+     *
+     * <p>Each user sends their own stream of OPUS encoded audio and each packet is assigned with a user id and SSRC.
+     * The decoder will be provided by JDA but need not be used.
+     *
+     * @return True, if {@link #handleEncodedAudio(OpusPacket)} should receive opus packets.
+     *
+     * @since  4.0.0
+     */
+    default boolean canReceiveEncoded()
+    {
+        return false;
+    }
+
+    /**
+     * If {@link #canReceiveEncoded()} returns true, JDA will provide raw {@link net.dv8tion.jda.api.audio.OpusPacket OpusPackets}
+     * to this method <b>every 20 milliseconds</b>. These packets are for specific users rather than a combined packet
+     * of all users like {@link #handleCombinedAudio(CombinedAudio)}.
+     *
+     * <p>This is useful for systems that want to either do lazy decoding of audio through {@link net.dv8tion.jda.api.audio.OpusPacket#getAudioData(double)}
+     * or for systems that can decode and transform the audio data manually without JDA involvement.
+     *
+     * @param packet
+     *        The {@link net.dv8tion.jda.api.audio.OpusPacket}
+     *
+     * @since  4.0.0
+     */
+    default void handleEncodedAudio(@Nonnull OpusPacket packet) {}
 
     /**
      * If {@link #canReceiveCombined()} returns true, JDA will provide a {@link net.dv8tion.jda.api.audio.CombinedAudio CombinedAudio}
@@ -65,7 +102,7 @@ public interface AudioReceiveHandler
      * @param  combinedAudio
      *         The combined audio data.
      */
-    void handleCombinedAudio(@Nonnull CombinedAudio combinedAudio);
+    default void handleCombinedAudio(@Nonnull CombinedAudio combinedAudio) {}
 
     /**
      * If {@link #canReceiveUser()} returns true, JDA will provide a {@link net.dv8tion.jda.api.audio.UserAudio UserAudio}
@@ -88,7 +125,7 @@ public interface AudioReceiveHandler
      * @param  userAudio
      *         The user audio data
      */
-    void handleUserAudio(@Nonnull UserAudio userAudio);
+    default void handleUserAudio(@Nonnull UserAudio userAudio) {}
 
     /**
      * This method is a filter predicate used by JDA to determine whether or not to include a

src/main/java/net/dv8tion/jda/api/audio/CombinedAudio.java

@@ -67,21 +67,6 @@ public List<User> getUsers()
     @Nonnull
     public byte[] getAudioData(double volume)
     {
-        short s;
-        int byteIndex = 0;
-        byte[] audio = new byte[audioData.length * 2];
-        for (int i = 0; i < audioData.length; i++)
-        {
-            s = audioData[i];
-            if (volume != 1.0)
-                s = (short) (s * volume);
-
-            byte leftByte = (byte) ((0x000000FF) & (s >> 8));
-            byte rightByte =  (byte) (0x000000FF & s);
-            audio[byteIndex] = leftByte;
-            audio[byteIndex + 1] = rightByte;
-            byteIndex += 2;
-        }
-        return audio;
+        return OpusPacket.getAudioData(audioData, volume);
     }
 }

src/main/java/net/dv8tion/jda/api/audio/OpusPacket.java

@@ -0,0 +1,243 @@
+/*
+ * Copyright 2015-2019 Austin Keener, Michael Ritter, Florian Spieß, and the JDA contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.dv8tion.jda.api.audio;
+
+import net.dv8tion.jda.internal.audio.AudioPacket;
+import net.dv8tion.jda.internal.audio.Decoder;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.util.Arrays;
+import java.util.Objects;
+
+/**
+ * A raw OPUS packet received from Discord that can be used for lazy decoding.
+ *
+ * @since  4.0.0
+ *
+ * @see AudioReceiveHandler#canReceiveEncoded()
+ * @see AudioReceiveHandler#handleEncodedAudio(OpusPacket)
+ */
+public final class OpusPacket implements Comparable<OpusPacket>
+{
+    /** (Hz) We want to use the highest of qualities! All the bandwidth! */
+    public static final int OPUS_SAMPLE_RATE = 48000;
+    /** An opus frame size of 960 at 48000hz represents 20 milliseconds of audio. */
+    public static final int OPUS_FRAME_SIZE = 960;
+    /** This is 20 milliseconds. We are only dealing with 20ms opus packets. */
+    public static final int OPUS_FRAME_TIME_AMOUNT = 20;
+    /** We want to use stereo. If the audio given is mono, the encoder promotes it to Left and Right mono (stereo that is the same on both sides) */
+    public static final int OPUS_CHANNEL_COUNT = 2;
+
+    private final long userId;
+    private final byte[] opusAudio;
+    private final Decoder decoder;
+    private final AudioPacket rawPacket;
+
+    private short[] decoded;
+    private boolean triedDecode;
+
+    public OpusPacket(@Nonnull AudioPacket packet, long userId, @Nullable Decoder decoder)
+    {
+        this.rawPacket = packet;
+        this.userId = userId;
+        this.decoder = decoder;
+        this.opusAudio = packet.getEncodedAudio().array();
+    }
+
+    /**
+     * The sequence number of this packet. This is used as ordering key for {@link #compareTo(OpusPacket)}.
+     * <br>A char represents an unsigned short value in this case.
+     *
+     * <p>Note that packet sequence is important for decoding. If a packet is out of sequence the decode
+     * step will fail.
+     *
+     * @return The sequence number of this packet
+     *
+     * @see    <a href="http://www.rfcreader.com/#rfc3550_line548" target="_blank">RTP Header</a>
+     */
+    public char getSequence()
+    {
+        return rawPacket.getSequence();
+    }
+
+    /**
+     * The timestamp for this packet. As specified by the RTP header.
+     *
+     * @return The timestamp
+     *
+     * @see    <a href="http://www.rfcreader.com/#rfc3550_line548" target="_blank">RTP Header</a>
+     */
+    public int getTimestamp()
+    {
+        return rawPacket.getTimestamp();
+    }
+
+    /**
+     * The synchronization source identifier (SSRC) for the user that sent this audio packet.
+     *
+     * @return The SSRC
+     *
+     * @see    <a href="http://www.rfcreader.com/#rfc3550_line548" target="_blank">RTP Header</a>
+     */
+    public int getSSRC()
+    {
+        return rawPacket.getSSRC();
+    }
+
+    /**
+     * The ID of the responsible {@link net.dv8tion.jda.api.entities.User}.
+     *
+     * @return The user id
+     */
+    public long getUserId()
+    {
+        return userId;
+    }
+
+    /**
+     * Whether {@link #decode()} is possible.
+     *
+     * @return True, if decode is possible.
+     */
+    public boolean canDecode()
+    {
+        return decoder != null && decoder.isInOrder(getSequence());
+    }
+
+    /**
+     * The raw opus audio, copied to a new array.
+     *
+     * @return The raw opus audio
+     */
+    @Nonnull
+    public byte[] getOpusAudio()
+    {
+        //prevent write access to backing array
+        return Arrays.copyOf(opusAudio, opusAudio.length);
+    }
+
+    /**
+     * Attempts to decode the opus packet.
+     * <br>This method is idempotent and will provide the same result on multiple calls
+     * without decoding again.
+     *
+     * For most use-cases {@link #getAudioData(double)} should be used instead.
+     *
+     * @throws java.lang.IllegalStateException
+     *         If {@link #canDecode()} is false
+     *
+     * @return The decoded audio or {@code null} if decoding failed for some reason.
+     *
+     * @see    #canDecode()
+     * @see    #getAudioData(double)
+     */
+    @Nullable
+    public synchronized short[] decode()
+    {
+        if (triedDecode)
+            return decoded;
+        if (decoder == null)
+            throw new IllegalStateException("No decoder available");
+        if (!decoder.isInOrder(getSequence()))
+            throw new IllegalStateException("Packet is not in order");
+        triedDecode = true;
+        return decoded = decoder.decodeFromOpus(rawPacket); // null if failed to decode
+    }
+
+    /**
+     * Decodes and adjusts the opus audio for the specified volume.
+     * <br>The provided volume should be a double precision floating point in the interval from 0 to 1.
+     * In this case 0.5 would represent 50% volume for instance.
+     *
+     * @param  volume
+     *         The volume
+     *
+     * @throws java.lang.IllegalArgumentException
+     *         If {@link #decode()} returns null
+     *
+     * @return The stereo PCM audio data as specified by {@link net.dv8tion.jda.api.audio.AudioReceiveHandler#OUTPUT_FORMAT}.
+     */
+    @Nonnull
+    @SuppressWarnings("ConstantConditions") // the null case is handled with an exception
+    public byte[] getAudioData(double volume)
+    {
+        return getAudioData(decode(), volume); // throws IllegalArgument if decode failed
+    }
+
+    /**
+     * Decodes and adjusts the opus audio for the specified volume.
+     * <br>The provided volume should be a double precision floating point in the interval from 0 to 1.
+     * In this case 0.5 would represent 50% volume for instance.
+     *
+     * @param  decoded
+     *         The decoded audio data
+     * @param  volume
+     *         The volume
+     *
+     * @throws java.lang.IllegalArgumentException
+     *         If {@code decoded} is null
+     *
+     * @return The stereo PCM audio data as specified by {@link net.dv8tion.jda.api.audio.AudioReceiveHandler#OUTPUT_FORMAT}.
+     */
+    @Nonnull
+    @SuppressWarnings("ConstantConditions") // the null case is handled with an exception
+    public static byte[] getAudioData(@Nonnull short[] decoded, double volume)
+    {
+        if (decoded == null)
+            throw new IllegalArgumentException("Cannot get audio data from null");
+        int byteIndex = 0;
+        byte[] audio = new byte[decoded.length * 2];
+        for (short s : decoded)
+        {
+            if (volume != 1.0)
+                s = (short) (s * volume);
+
+            byte leftByte  = (byte) ((s >>> 8) & 0xFF);
+            byte rightByte = (byte)  (s        & 0xFF);
+            audio[byteIndex] = leftByte;
+            audio[byteIndex + 1] = rightByte;
+            byteIndex += 2;
+        }
+        return audio;
+    }
+
+    @Override
+    public int compareTo(@Nonnull OpusPacket o)
+    {
+        return getSequence() - o.getSequence();
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hash(getSequence(), getTimestamp(), getOpusAudio());
+    }
+
+    @Override
+    public boolean equals(Object obj)
+    {
+        if (obj == this)
+            return true;
+        if (!(obj instanceof OpusPacket))
+            return false;
+        OpusPacket other = (OpusPacket) obj;
+        return getSequence() == other.getSequence()
+            && getTimestamp() == other.getTimestamp()
+            && getSsrc() == other.getSsrc();
+    }
+}