Support for decoding & rendering of video streams

* Adds support for rendering video streams to graphics backends; D3D11, D3D12, and
* Adds support for cross-platform/device video decoding via ffmpeg (NVDEC / CUVID / D3D11VA / VAAPI / CPU)
* Adds support for video decoding via MediaCodec for Android platforms
* Adds support for graphics api buffer interop with CUDA (NVDEC,CUVID), D3D11VA, and MediaCodec
* Adds support for handling more packet types, (Guardian change, session update, etc)
* Adds support for Pico Neo 3 OpenXR runtime/loader
* Adds support for sending Stats
* Adds support for using latency/timing data for pose predication.
* Adds support for using FB OpenXR extension for colospaces.
* Adds new command line options.
* Updated xtask build scripts.
* Some placeholder work for building AppImages on Linux (disabled/not working).
* Updates Oculus Mobile OXR SDK to v39
* Refactor controller updates & hmd poses to tracking update thread.
* Converts various struct init code to C99/C++20 designated initializer form for improved readability and constant,
* Various code clean-up and bug fixes.

Cargo.toml

@@ -17,7 +17,9 @@ members = [
     "alvr/openxr-client/alxr-engine-sys",
     "alvr/openxr-client/alxr-client",
     "alvr/openxr-client/alxr-android-client",
+    "alvr/openxr-client/alxr-android-client/quest",
+    "alvr/openxr-client/alxr-android-client/pico-neo",
     "alvr/xtask",
-    "alvr/experiments/client",     # needed for bug in cargo-subcommand
-    "alvr/experiments/*",
+    #"alvr/experiments/client",     # needed for bug in cargo-subcommand
+    #"alvr/experiments/*",
 ]

alvr/client/android/app/src/main/cpp/fec.cpp

@@ -4,19 +4,25 @@
 #include <inttypes.h>
 #include "fec.h"
 #include "packet_types.h"
+#ifndef ALXR_CLIENT
 #include "utils.h"
+#else
+#include <cstring>
+namespace {
+    inline void LOGI(...) {}
+    inline void LOGE(...) {}
+    inline void FrameLog(uint64_t /*frameIndex*/, const char*/*format*/, ...) {}
+}
+#endif
 
-bool FECQueue::reed_solomon_initialized = false;
+std::once_flag FECQueue::reed_solomon_initialized{};
 
 FECQueue::FECQueue() {
     m_currentFrame.videoFrameIndex = UINT64_MAX;
     m_recovered = true;
     m_fecFailure = false;
 
-    if (!reed_solomon_initialized) {
-        reed_solomon_init();
-        reed_solomon_initialized = true;
-    }
+    std::call_once(reed_solomon_initialized, reed_solomon_init);
 }
 
 FECQueue::~FECQueue() {

alvr/client/android/app/src/main/cpp/fec.h

@@ -3,6 +3,7 @@
 
 #include <list>
 #include <vector>
+#include <mutex>
 #include "packet_types.h"
 #include "reedsolomon/rs.h"
 
@@ -37,7 +38,7 @@ class FECQueue {
     bool m_fecFailure;
     reed_solomon *m_rs = NULL;
 
-    static bool reed_solomon_initialized;
+    static std::once_flag reed_solomon_initialized;
 };
 
 #endif //ALVRCLIENT_FEC_H

alvr/client/android/app/src/main/cpp/latency_collector.cpp

@@ -1,7 +1,40 @@
-#include <jni.h>
 #include "latency_collector.h"
-#include "utils.h"
-#include "bindings.h"
+#include <chrono>
+#ifndef ALXR_CLIENT
+    #include "utils.h"
+    #include "bindings.h"
+#endif
+
+namespace {
+#if 0//def __linux__
+    struct XrSteadyClock : std::chrono::steady_clock
+    {
+        using time_point = std::chrono::time_point<XrSteadyClock, duration>;
+        constexpr static const bool is_steady = true;
+        static inline time_point now() noexcept
+        {
+            using namespace std::chrono;
+            struct timespec ts;
+            clock_gettime(CLOCK_BOOTTIME, &ts);
+            return time_point(seconds(ts.tv_sec) + nanoseconds(ts.tv_nsec));
+        }
+    };
+#else
+    using XrSteadyClock = std::chrono::steady_clock;
+#endif
+    constexpr const std::uint64_t USECS_IN_SEC = 1000 * 1000;
+
+    inline std::uint64_t getTimestampUs() {
+        using namespace std::chrono;
+        using ClockType = XrSteadyClock;
+        static_assert(ClockType::is_steady);
+        using microsecondsU64 = duration<std::uint64_t, microseconds::period>;
+        return duration_cast<microsecondsU64>(ClockType::now().time_since_epoch()).count();
+    }
+#ifdef ALXR_CLIENT
+    inline void FrameLog(...) {}
+#endif
+}
 
 LatencyCollector LatencyCollector::m_Instance;
 
@@ -21,7 +54,7 @@ LatencyCollector::FrameTimestamp &LatencyCollector::getFrame(uint64_t frameIndex
 
 void LatencyCollector::setTotalLatency(uint32_t latency) {
     if (latency < 2e5)
-        m_ServerTotalLatency = latency * 0.05 + m_ServerTotalLatency * 0.95;
+        m_ServerTotalLatency.store(latency * 0.05 + m_ServerTotalLatency.load() * 0.95);
 }
 void LatencyCollector::tracking(uint64_t frameIndex) {
     getFrame(frameIndex).tracking = getTimestampUs();
@@ -76,11 +109,12 @@ void LatencyCollector::submit(uint64_t frameIndex) {
 
     m_FramesInSecond = 1000000.0 / (timestamp.submit - m_LastSubmit);
     m_LastSubmit = timestamp.submit;
-
+#ifndef NDEBUG
     FrameLog(frameIndex, "totalLatency=%.1f transportLatency=%.1f decodeLatency=%.1f renderLatency1=%.1f renderLatency2=%.1f"
             , m_Latency[0] / 1000.0, m_Latency[1] / 1000.0, m_Latency[2] / 1000.0
             , (timestamp.rendered2 - timestamp.decoderOutput) / 1000.0
             , (timestamp.submit - timestamp.rendered2) / 1000.0);
+#endif
 }
 
 void LatencyCollector::resetAll() {
@@ -135,29 +169,27 @@ void LatencyCollector::submitNewFrame() {
     checkAndResetSecond();
 }
 
-uint64_t LatencyCollector::getTrackingPredictionLatency() {
-    if (m_ServerTotalLatency > 2e5)
-        return 2e5;
-    else
-        return m_ServerTotalLatency;
+uint64_t LatencyCollector::getTrackingPredictionLatency() const {
+    const auto predictionLatency = m_ServerTotalLatency.load();
+    return predictionLatency > 2e5 ? 2e5 : predictionLatency;
 }
 
-uint64_t LatencyCollector::getLatency(uint32_t i) {
+uint64_t LatencyCollector::getLatency(uint32_t i) const {
     return m_Latency[i];
 }
-uint64_t LatencyCollector::getPacketsLostTotal() {
+uint64_t LatencyCollector::getPacketsLostTotal() const {
     return m_PacketsLostTotal;
 }
-uint64_t LatencyCollector::getPacketsLostInSecond() {
+uint64_t LatencyCollector::getPacketsLostInSecond() const {
     return m_PacketsLostPrevious;
 }
-uint64_t LatencyCollector::getFecFailureTotal() {
+uint64_t LatencyCollector::getFecFailureTotal() const {
     return m_FecFailureTotal;
 }
-uint64_t LatencyCollector::getFecFailureInSecond() {
+uint64_t LatencyCollector::getFecFailureInSecond() const {
     return m_FecFailurePrevious;
 }
-float LatencyCollector::getFramesInSecond() {
+float LatencyCollector::getFramesInSecond() const {
     return m_FramesInSecond;
 }
 

alvr/client/android/app/src/main/cpp/latency_collector.h

@@ -4,19 +4,20 @@
 #include <memory>
 #include <vector>
 #include <map>
+#include <atomic>
 #include <mutex>
 
 class LatencyCollector {
 public:
     static LatencyCollector &Instance();
 
-    uint64_t getTrackingPredictionLatency();
-    uint64_t getLatency(uint32_t i);
-    uint64_t getPacketsLostTotal();
-    uint64_t getPacketsLostInSecond();
-    uint64_t getFecFailureTotal();
-    uint64_t getFecFailureInSecond();
-    float getFramesInSecond();
+    uint64_t getTrackingPredictionLatency() const;
+    uint64_t getLatency(uint32_t i) const;
+    uint64_t getPacketsLostTotal() const;
+    uint64_t getPacketsLostInSecond() const;
+    uint64_t getFecFailureTotal() const;
+    uint64_t getFecFailureInSecond() const;
+    float getFramesInSecond() const;
 
     void packetLoss(int64_t lost);
     void fecFailure();
@@ -72,7 +73,7 @@ class LatencyCollector {
     uint64_t m_FecFailureInSecond = 0;
     uint64_t m_FecFailurePrevious = 0;
 
-    uint32_t m_ServerTotalLatency = 0;
+    std::atomic<uint32_t> m_ServerTotalLatency { 0 };
 
     // Total/Transport/Decode/Idle latency
     uint64_t m_Latency[5];

alvr/filesystem/src/lib.rs

@@ -67,26 +67,26 @@ pub fn server_build_dir() -> PathBuf {
     build_dir().join(server_build_dir)
 }
 
-pub fn alxr_android_build_dir() -> PathBuf {
-    build_dir().join("alxr_client_android")
+pub fn alxr_android_build_dir< P: AsRef<Path> >(prefix: P) -> PathBuf {
+    build_dir().join(prefix).join("alxr_client_android")
 }
 
-pub fn alxr_client_build_dir() -> PathBuf {
+pub fn alxr_client_build_dir< P: AsRef<Path> >(prefix: P, cuda_enabled: bool) -> PathBuf {
     if cfg!(target_os = "android") {
-        return alxr_android_build_dir();
+        return alxr_android_build_dir(prefix);
     }
 
     let alxr_client_dir = if cfg!(windows) {
-        "alxr_client_windows"
+        if cuda_enabled { "alxr_client_windows_cuda" } else { "alxr_client_windows" }
     } else if cfg!(target_os = "linux") {
-        "alxr_client_linux"
+        if cuda_enabled { "alxr_client_linux_cuda" } else { "alxr_client_linux" }
     } else if cfg!(target_os = "macos") {
         "alxr_client_macos"
     } else {
         unimplemented!()
     };
 
-    build_dir().join(alxr_client_dir)
+    build_dir().join(prefix).join(alxr_client_dir)
 }
 
 pub fn installer_path() -> PathBuf {