diff --git a/CMakeLists.txt b/CMakeLists.txt index 39d06fa..e64fae0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -77,6 +77,7 @@ set(FFI_PROTO_FILES ${FFI_PROTO_DIR}/e2ee.proto ${FFI_PROTO_DIR}/stats.proto ${FFI_PROTO_DIR}/data_stream.proto + ${FFI_PROTO_DIR}/data_track.proto ${FFI_PROTO_DIR}/rpc.proto ${FFI_PROTO_DIR}/track_publication.proto ) @@ -325,6 +326,7 @@ add_library(livekit SHARED src/audio_source.cpp src/audio_stream.cpp src/data_stream.cpp + src/data_track_subscription.cpp src/e2ee.cpp src/ffi_handle.cpp src/ffi_client.cpp @@ -332,7 +334,9 @@ add_library(livekit SHARED src/livekit.cpp src/logging.cpp src/local_audio_track.cpp + src/local_data_track.cpp src/remote_audio_track.cpp + src/remote_data_track.cpp src/room.cpp src/room_proto_converter.cpp src/room_proto_converter.h @@ -682,10 +686,6 @@ install(FILES # Build the LiveKit C++ bridge before examples (human_robot depends on it) add_subdirectory(bridge) -# ---- Examples ---- -# add_subdirectory(examples) - - if(LIVEKIT_BUILD_EXAMPLES) add_subdirectory(examples) endif() diff --git a/README.md b/README.md index eb473f2..97fc77f 100644 --- a/README.md +++ b/README.md @@ -447,6 +447,37 @@ CPP SDK is using clang C++ format brew install clang-format ``` + +#### Memory Checks +Run valgrind on various examples or tests to check for memory leaks and other issues. +```bash +valgrind --leak-check=full ./build-debug/bin/BridgeRobot +valgrind --leak-check=full ./build-debug/bin/BridgeHuman +valgrind --leak-check=full ./build-debug/bin/livekit_integration_tests +valgrind --leak-check=full ./build-debug/bin/livekit_stress_tests +``` + +# Running locally +1. Install the livekit-server +https://docs.livekit.io/transport/self-hosting/local/ + +Start the livekit-server with data tracks enabled: +```bash +LIVEKIT_CONFIG="enable_data_tracks: true" livekit-server --dev +``` + +```bash +# generate tokens, do for all participants +lk token create \ + --api-key devkey \ + --api-secret secret \ + -i robot \ + --join \ + --valid-for 99999h \ + --room robo_room \ + --grant '{"canPublish":true,"canSubscribe":true,"canPublishData":true}' +``` +
diff --git a/bridge/CMakeLists.txt b/bridge/CMakeLists.txt index c1b2369..996b82e 100644 --- a/bridge/CMakeLists.txt +++ b/bridge/CMakeLists.txt @@ -8,6 +8,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) add_library(livekit_bridge SHARED src/livekit_bridge.cpp src/bridge_audio_track.cpp + src/bridge_data_track.cpp src/bridge_video_track.cpp src/bridge_room_delegate.cpp src/bridge_room_delegate.h diff --git a/bridge/README.md b/bridge/README.md index 76b4f2b..f77da63 100644 --- a/bridge/README.md +++ b/bridge/README.md @@ -206,10 +206,7 @@ The human will print periodic summaries like: ## Testing -The bridge includes a unit test suite built with [Google Test](https://github.com/google/googletest). Tests cover -1. `CallbackKey` hashing/equality, -2. `BridgeAudioTrack`/`BridgeVideoTrack` state management, and -3. `LiveKitBridge` pre-connection behaviour (callback registration, error handling). +The bridge includes a unit test suite built with [Google Test](https://github.com/google/googletest). ### Building and running tests @@ -222,18 +219,46 @@ Bridge tests are automatically included when you build with the `debug-tests` or Then run them directly: ```bash -./build-debug/bin/livekit_bridge_tests +./build-debug/bin/livekit_bridge_*_tests ``` -### Standalone bridge tests only +### Bridge Tests -If you want to build bridge tests independently (without the parent SDK tests), set `LIVEKIT_BRIDGE_BUILD_TESTS=ON`: +The bridge layer (`bridge/tests/`) has its own integration and stress tests +that exercise the full `LiveKitBridge` API over a real LiveKit server. +They use the **same environment variables** as the SDK tests above. ```bash -cmake --preset macos-debug -DLIVEKIT_BRIDGE_BUILD_TESTS=ON -cmake --build build-debug --target livekit_bridge_tests +# Run bridge tests via CTest +cd build-debug && ctest -L "bridge_integration" --output-on-failure +cd build-debug && ctest -L "bridge_stress" --output-on-failure + +# Or run executables directly +./build-debug/bin/livekit_bridge_integration_tests +./build-debug/bin/livekit_bridge_stress_tests + +# Run specific test suites +./build-debug/bin/livekit_bridge_integration_tests --gtest_filter="*AudioFrameRoundTrip*" +./build-debug/bin/livekit_bridge_stress_tests --gtest_filter="*HighThroughput*" ``` +| Executable | Description | +|------------|-------------| +| `livekit_bridge_unit_tests` | Unit tests (no server required) | +| `livekit_bridge_integration_tests` | Audio & data round-trip, latency, connect/disconnect cycles | +| `livekit_bridge_stress_tests` | Sustained push, lifecycle, callback churn, multi-track concurrency | + +#### Stress test suites + +| Test file | Tests | What it exercises | +|-----------|-------|-------------------| +| `test_bridge_audio_stress` | SustainedAudioPush, ReleaseUnderActivePush, RapidConnectDisconnectWithCallback | Audio push at real-time pace, release/push race, abrupt teardown | +| `test_bridge_data_stress` | HighThroughput, LargePayloadStress, CallbackChurn | Data throughput, 64 KB frames, data callback register/clear churn | +| `test_bridge_lifecycle_stress` | DisconnectUnderLoad, TrackReleaseWhileReceiving, FullLifecycleSoak | Disconnect with active pushers+receivers, mid-stream track release, repeated full create→push→release→destroy | +| `test_bridge_callback_stress` | AudioCallbackChurn, MixedCallbackChurn, CallbackReplacement | Audio set/clear churn, mixed audio+data churn, rapid callback overwrite | +| `test_bridge_multi_track_stress` | ConcurrentMultiTrackPush, ConcurrentCreateRelease, FullDuplexMultiTrack | 4 tracks pushed concurrently, create/release cycles under contention, bidirectional audio+data | + + ## Limitations The bridge is designed for simplicity and currently only supports limited audio and video features. It does not expose: diff --git a/bridge/include/livekit_bridge/bridge_audio_track.h b/bridge/include/livekit_bridge/bridge_audio_track.h index 5683e06..c313c0a 100644 --- a/bridge/include/livekit_bridge/bridge_audio_track.h +++ b/bridge/include/livekit_bridge/bridge_audio_track.h @@ -1,5 +1,5 @@ /* - * Copyright 2025 LiveKit + * Copyright 2026 LiveKit * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -55,6 +55,10 @@ class BridgeAudioTrackTest; * one thread while another calls mute()/unmute()/release(), or to call * pushFrame() concurrently from multiple threads. * + * All public methods are thread-safe: it is safe to call pushFrame() from + * one thread while another calls mute()/unmute()/release(), or to call + * pushFrame() concurrently from multiple threads. + * * Usage: * auto mic = bridge.createAudioTrack("mic", 48000, 2, * livekit::TrackSource::SOURCE_MICROPHONE); diff --git a/bridge/include/livekit_bridge/bridge_data_track.h b/bridge/include/livekit_bridge/bridge_data_track.h new file mode 100644 index 0000000..e009f95 --- /dev/null +++ b/bridge/include/livekit_bridge/bridge_data_track.h @@ -0,0 +1,127 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace livekit { +class LocalDataTrack; +class LocalParticipant; +} // namespace livekit + +namespace livekit_bridge { + +namespace test { +class BridgeDataTrackTest; +} // namespace test + +/** + * Handle to a published local data track. + * + * Created via LiveKitBridge::createDataTrack(). The bridge retains a + * reference to every track it creates and will automatically release all + * tracks when disconnect() is called. To unpublish a track mid-session, + * call release() explicitly. + * + * Unlike BridgeAudioTrack / BridgeVideoTrack, data tracks have no + * Source, Publication, mute(), or unmute(). They carry arbitrary binary + * frames via pushFrame(). + * + * All public methods are thread-safe. + * + * Usage: + * auto dt = bridge.createDataTrack("sensor-data"); + * dt->pushFrame({0x01, 0x02, 0x03}); + * dt->release(); // unpublishes mid-session + */ +class BridgeDataTrack { +public: + ~BridgeDataTrack(); + + BridgeDataTrack(const BridgeDataTrack &) = delete; + BridgeDataTrack &operator=(const BridgeDataTrack &) = delete; + + /** + * Push a binary frame to all subscribers of this data track. + * + * @param payload Raw bytes to send. + * @param user_timestamp Optional application-defined timestamp. + * @return true if the frame was pushed, false if the track has been + * released or the push failed (e.g. back-pressure). + */ + bool pushFrame(const std::vector &payload, + std::optional user_timestamp = std::nullopt); + + /** + * Push a binary frame from a raw pointer. + * + * @param data Pointer to raw bytes. + * @param size Number of bytes. + * @param user_timestamp Optional application-defined timestamp. + * @return true on success, false if released or push failed. + */ + bool pushFrame(const std::uint8_t *data, std::size_t size, + std::optional user_timestamp = std::nullopt); + + /// Track name as provided at creation. + const std::string &name() const noexcept { return name_; } + + /// Whether the track is still published in the room. + bool isPublished() const; + + /// Whether this track has been released / unpublished. + bool isReleased() const noexcept; + + /** + * Explicitly unpublish the track and release underlying SDK resources. + * + * After this call, pushFrame() returns false. Called automatically by the + * destructor and by LiveKitBridge::disconnect(). Safe to call multiple + * times (idempotent). + */ + void release(); + +private: + friend class LiveKitBridge; + friend class test::BridgeDataTrackTest; + + BridgeDataTrack(std::string name, + std::shared_ptr track, + livekit::LocalParticipant *participant); + + /** Protects released_ and track_ for thread-safe access. */ + mutable std::mutex mutex_; + + /** Publisher-assigned track name (immutable after construction). */ + std::string name_; + + /** True after release() or disconnect(); prevents further pushFrame(). */ + bool released_ = false; + + /** Underlying SDK data track handle. Nulled on release(). */ + std::shared_ptr track_; + + /** Participant that published this track; used for unpublish. Not owned. */ + livekit::LocalParticipant *participant_ = nullptr; +}; + +} // namespace livekit_bridge diff --git a/bridge/include/livekit_bridge/bridge_video_track.h b/bridge/include/livekit_bridge/bridge_video_track.h index 8057b7a..350c852 100644 --- a/bridge/include/livekit_bridge/bridge_video_track.h +++ b/bridge/include/livekit_bridge/bridge_video_track.h @@ -1,5 +1,5 @@ /* - * Copyright 2025 LiveKit + * Copyright 2026 LiveKit * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -55,6 +55,10 @@ class BridgeVideoTrackTest; * one thread while another calls mute()/unmute()/release(), or to call * pushFrame() concurrently from multiple threads. * + * All public methods are thread-safe: it is safe to call pushFrame() from + * one thread while another calls mute()/unmute()/release(), or to call + * pushFrame() concurrently from multiple threads. + * * Usage: * auto cam = bridge.createVideoTrack("cam", 1280, 720, * livekit::TrackSource::SOURCE_CAMERA); diff --git a/bridge/include/livekit_bridge/livekit_bridge.h b/bridge/include/livekit_bridge/livekit_bridge.h index 3d27dea..536b1f1 100644 --- a/bridge/include/livekit_bridge/livekit_bridge.h +++ b/bridge/include/livekit_bridge/livekit_bridge.h @@ -1,5 +1,5 @@ /* - * Copyright 2025 LiveKit + * Copyright 2026 LiveKit * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ #pragma once #include "livekit_bridge/bridge_audio_track.h" +#include "livekit_bridge/bridge_data_track.h" #include "livekit_bridge/bridge_video_track.h" #include "livekit/room.h" @@ -28,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -39,6 +41,8 @@ class AudioFrame; class VideoFrame; class AudioStream; class VideoStream; +class DataTrackSubscription; +class RemoteDataTrack; class Track; enum class TrackSource; } // namespace livekit @@ -63,6 +67,14 @@ using AudioFrameCallback = std::function; using VideoFrameCallback = std::function; +/// Callback type for incoming data track frames. +/// Called on a background reader thread. +/// @param payload Raw binary data received. +/// @param user_timestamp Optional application-defined timestamp from sender. +using DataFrameCallback = + std::function &payload, + std::optional user_timestamp)>; + /** * High-level bridge to the LiveKit C++ SDK. * @@ -165,6 +177,9 @@ class LiveKitBridge { * @pre The bridge must be connected (via connect()). Calling this on a * disconnected bridge is a programming error. * + * @pre The bridge must be connected (via connect()). Calling this on a + * disconnected bridge is a programming error. + * * @param name Human-readable track name. * @param sample_rate Sample rate in Hz (e.g. 48000). * @param num_channels Number of audio channels (1 = mono, 2 = stereo). @@ -189,6 +204,9 @@ class LiveKitBridge { * @pre The bridge must be connected (via connect()). Calling this on a * disconnected bridge is a programming error. * + * @pre The bridge must be connected (via connect()). Calling this on a + * disconnected bridge is a programming error. + * * @param name Human-readable track name. * @param width Video width in pixels. * @param height Video height in pixels. @@ -203,6 +221,19 @@ class LiveKitBridge { createVideoTrack(const std::string &name, int width, int height, livekit::TrackSource source); + /** + * Create and publish a local data track. + * + * Data tracks carry arbitrary binary frames and are independent of the + * audio/video track lifecycle. The bridge retains a reference and will + * automatically release on disconnect(). + * + * @param name Unique track name visible to other participants. + * @return Shared pointer to the published data track handle (never null). + * @throws std::runtime_error if the bridge is not connected. + */ + std::shared_ptr createDataTrack(const std::string &name); + // --------------------------------------------------------------- // Incoming frame callbacks // --------------------------------------------------------------- @@ -264,6 +295,37 @@ class LiveKitBridge { void clearOnVideoFrameCallback(const std::string &participant_identity, livekit::TrackSource source); + /** + * Set the callback for data frames from a specific remote participant's + * data track. + * + * The callback fires on a background thread whenever a new data frame is + * received. If the remote data track has not yet been published, the + * callback is stored and auto-wired when the track is published (via + * onRemoteDataTrackPublished). If the track was already published, the + * reader is started immediately—mirroring the onTrackSubscribed behavior + * for audio/video. + * + * Data tracks are keyed by (participant_identity, track_name) rather + * than TrackSource, since data tracks don't have a TrackSource enum. + * + * @param participant_identity Identity of the remote participant. + * @param track_name Name of the remote data track. + * @param callback Function to invoke per data frame. + */ + void setOnDataFrameCallback(const std::string &participant_identity, + const std::string &track_name, + DataFrameCallback callback); + + /** + * Clear the data frame callback for a specific remote participant + track + * name. + * + * If a reader thread is active, it is stopped and joined. + */ + void clearOnDataFrameCallback(const std::string &participant_identity, + const std::string &track_name); + private: friend class BridgeRoomDelegate; friend class test::CallbackKeyTest; @@ -290,7 +352,42 @@ class LiveKitBridge { bool is_audio = false; }; - /// Called by BridgeRoomDelegate when a remote track is subscribed. + /** + * Composite key for data track callbacks: (participant_identity, track_name). + * + * Data tracks are identified by name rather than TrackSource because they + * don't belong to the standard Source/Publication hierarchy used by + * audio/video tracks. + */ + struct DataCallbackKey { + /** Remote participant identity string. */ + std::string identity; + + /** Publisher-assigned data track name. */ + std::string track_name; + + bool operator==(const DataCallbackKey &o) const; + }; + + struct DataCallbackKeyHash { + std::size_t operator()(const DataCallbackKey &k) const; + }; + + /** Active reader thread + subscription for an incoming data track. */ + struct ActiveDataReader { + /** The remote track must stay alive for the subscription to receive frames. + * Dropping the RemoteDataTrack handle tells the Rust FFI we no longer care + * about this track, which may cause it to stop forwarding frames. */ + std::shared_ptr remote_track; + + /** Underlying SDK subscription that delivers frames via read(). */ + std::shared_ptr subscription; + + /** Background thread running the blocking read loop. */ + std::thread thread; + }; + + // Called by BridgeRoomDelegate when a remote track is subscribed void onTrackSubscribed(const std::string &participant_identity, livekit::TrackSource source, const std::shared_ptr &track); @@ -299,9 +396,17 @@ class LiveKitBridge { void onTrackUnsubscribed(const std::string &participant_identity, livekit::TrackSource source); - /// Extract the thread for the given callback key. - /// @pre Caller must hold @c mutex_. + // Called by BridgeRoomDelegate when a remote data track is published. + // If a callback is registered for (identity, track_name), starts the data + // reader thread (like onTrackSubscribed for audio/video); otherwise stores + // the track as pending until setOnDataFrameCallback is called. + void + onRemoteDataTrackPublished(std::shared_ptr track); + + /// Close the stream and extract the thread for the caller to join + /// (caller must hold mutex_) std::thread extractReaderThread(const CallbackKey &key); + std::thread extractDataReaderThread(const DataCallbackKey &key); /// Start a reader thread for a subscribed track. /// @return The reader thread for this track. @@ -313,6 +418,10 @@ class LiveKitBridge { std::thread startVideoReader(const CallbackKey &key, const std::shared_ptr &track, VideoFrameCallback cb); + std::thread + startDataReader(const DataCallbackKey &key, + const std::shared_ptr &track, + DataFrameCallback cb); mutable std::mutex mutex_; bool connected_; @@ -330,10 +439,22 @@ class LiveKitBridge { /// @copydoc audio_callbacks_ std::unordered_map video_callbacks_; + std::unordered_map + data_callbacks_; + + /// Remote data tracks published before a frame callback was registered; + /// when setOnDataFrameCallback is called for a matching key, we start the + /// reader and remove the track from here. + std::unordered_map, + DataCallbackKeyHash> + pending_remote_data_tracks_; - /// Active reader threads for subscribed tracks. + /// Active reader threads for subscribed audio and video tracks. std::unordered_map - active_readers_; + active_av_readers_; + /// Active reader threads for subscribed data tracks. + std::unordered_map + active_data_readers_; /// All tracks created by this bridge. The bridge retains a shared_ptr so /// it can force-release every track on disconnect() before the room is @@ -341,6 +462,7 @@ class LiveKitBridge { std::vector> published_audio_tracks_; /// @copydoc published_audio_tracks_ std::vector> published_video_tracks_; + std::vector> published_data_tracks_; }; } // namespace livekit_bridge diff --git a/bridge/src/bridge_audio_track.cpp b/bridge/src/bridge_audio_track.cpp index 654129c..230b733 100644 --- a/bridge/src/bridge_audio_track.cpp +++ b/bridge/src/bridge_audio_track.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2025 LiveKit + * Copyright 2026 LiveKit * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/bridge/src/bridge_data_track.cpp b/bridge/src/bridge_data_track.cpp new file mode 100644 index 0000000..66f4887 --- /dev/null +++ b/bridge/src/bridge_data_track.cpp @@ -0,0 +1,106 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "livekit_bridge/bridge_data_track.h" + +#include "lk_log.h" + +#include "livekit/data_frame.h" +#include "livekit/local_data_track.h" +#include "livekit/local_participant.h" + +namespace livekit_bridge { + +BridgeDataTrack::BridgeDataTrack(std::string name, + std::shared_ptr track, + livekit::LocalParticipant *participant) + : name_(std::move(name)), track_(std::move(track)), + participant_(participant) {} + +BridgeDataTrack::~BridgeDataTrack() { release(); } + +bool BridgeDataTrack::pushFrame(const std::vector &payload, + std::optional user_timestamp) { + std::lock_guard lock(mutex_); + if (released_ || !track_) { + return false; + } + + livekit::DataFrame frame; + frame.payload = payload; + frame.user_timestamp = user_timestamp; + + try { + return track_->tryPush(frame); + } catch (const std::exception &e) { + LK_LOG_ERROR("[BridgeDataTrack] tryPush error: {}", e.what()); + return false; + } +} + +bool BridgeDataTrack::pushFrame(const std::uint8_t *data, std::size_t size, + std::optional user_timestamp) { + std::lock_guard lock(mutex_); + if (released_ || !track_) { + return false; + } + + livekit::DataFrame frame; + frame.payload.assign(data, data + size); + frame.user_timestamp = user_timestamp; + + try { + return track_->tryPush(frame); + } catch (const std::exception &e) { + LK_LOG_ERROR("[BridgeDataTrack] tryPush error: {}", e.what()); + return false; + } +} + +bool BridgeDataTrack::isPublished() const { + std::lock_guard lock(mutex_); + if (released_ || !track_) { + return false; + } + return track_->isPublished(); +} + +bool BridgeDataTrack::isReleased() const noexcept { + std::lock_guard lock(mutex_); + return released_; +} + +void BridgeDataTrack::release() { + std::lock_guard lock(mutex_); + if (released_) { + return; + } + released_ = true; + + if (participant_ && track_) { + try { + participant_->unpublishDataTrack(track_); + } catch (...) { + LK_LOG_ERROR("[BridgeDataTrack] unpublishDataTrack error, continuing " + "with cleanup"); + } + } + + track_.reset(); + participant_ = nullptr; +} + +} // namespace livekit_bridge diff --git a/bridge/src/bridge_room_delegate.cpp b/bridge/src/bridge_room_delegate.cpp index 118627c..2f563eb 100644 --- a/bridge/src/bridge_room_delegate.cpp +++ b/bridge/src/bridge_room_delegate.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2025 LiveKit + * Copyright 2026 LiveKit * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,9 @@ #include "bridge_room_delegate.h" +#include "lk_log.h" + +#include "livekit/remote_data_track.h" #include "livekit/remote_participant.h" #include "livekit/remote_track_publication.h" #include "livekit/track.h" @@ -50,4 +53,19 @@ void BridgeRoomDelegate::onTrackUnsubscribed( bridge_.onTrackUnsubscribed(identity, source); } +void BridgeRoomDelegate::onRemoteDataTrackPublished( + livekit::Room & /*room*/, + const livekit::RemoteDataTrackPublishedEvent &ev) { + if (!ev.track) { + LK_LOG_ERROR("[BridgeRoomDelegate] onRemoteDataTrackPublished called " + "with null track."); + return; + } + + LK_LOG_INFO("[BridgeRoomDelegate] onRemoteDataTrackPublished: \"{}\" from " + "\"{}\"", + ev.track->info().name, ev.track->publisherIdentity()); + bridge_.onRemoteDataTrackPublished(ev.track); +} + } // namespace livekit_bridge diff --git a/bridge/src/bridge_room_delegate.h b/bridge/src/bridge_room_delegate.h index 8f5dd31..9c5d0da 100644 --- a/bridge/src/bridge_room_delegate.h +++ b/bridge/src/bridge_room_delegate.h @@ -1,5 +1,5 @@ /* - * Copyright 2025 LiveKit + * Copyright 2026 LiveKit * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -44,6 +44,10 @@ class BridgeRoomDelegate : public livekit::RoomDelegate { void onTrackUnsubscribed(livekit::Room &room, const livekit::TrackUnsubscribedEvent &ev) override; + void onRemoteDataTrackPublished( + livekit::Room &room, + const livekit::RemoteDataTrackPublishedEvent &ev) override; + private: LiveKitBridge &bridge_; }; diff --git a/bridge/src/bridge_video_track.cpp b/bridge/src/bridge_video_track.cpp index d78d232..0034612 100644 --- a/bridge/src/bridge_video_track.cpp +++ b/bridge/src/bridge_video_track.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2025 LiveKit + * Copyright 2026 LiveKit * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/bridge/src/livekit_bridge.cpp b/bridge/src/livekit_bridge.cpp index bae25c1..abf54fd 100644 --- a/bridge/src/livekit_bridge.cpp +++ b/bridge/src/livekit_bridge.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2025 LiveKit + * Copyright 2026 LiveKit * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,11 +23,15 @@ #include "livekit/audio_frame.h" #include "livekit/audio_source.h" #include "livekit/audio_stream.h" +#include "livekit/data_frame.h" +#include "livekit/data_track_subscription.h" #include "livekit/livekit.h" #include "livekit/local_audio_track.h" +#include "livekit/local_data_track.h" #include "livekit/local_participant.h" #include "livekit/local_track_publication.h" #include "livekit/local_video_track.h" +#include "livekit/remote_data_track.h" #include "livekit/room.h" #include "livekit/track.h" #include "livekit/video_frame.h" @@ -56,6 +60,18 @@ LiveKitBridge::CallbackKeyHash::operator()(const CallbackKey &k) const { return h1 ^ (h2 << 1); } +bool LiveKitBridge::DataCallbackKey::operator==( + const DataCallbackKey &o) const { + return identity == o.identity && track_name == o.track_name; +} + +std::size_t +LiveKitBridge::DataCallbackKeyHash::operator()(const DataCallbackKey &k) const { + std::size_t h1 = std::hash{}(k.identity); + std::size_t h2 = std::hash{}(k.track_name); + return h1 ^ (h2 << 1); +} + // --------------------------------------------------------------- // Construction / Destruction // --------------------------------------------------------------- @@ -92,12 +108,18 @@ bool LiveKitBridge::connect(const std::string &url, const std::string &token, } } - // ---- Phase 2: create room and connect without holding the lock ---- - // This avoids blocking other threads during the network handshake and - // eliminates the risk of deadlock if the SDK delivers delegate callbacks + // ---- Phase 2: create room, attach delegate, and connect without holding the + // lock ---- This avoids blocking other threads during the network handshake + // and eliminates the risk of deadlock if the SDK delivers delegate callbacks // synchronously during Connect(). + // Set the delegate before Connect() so that events (e.g. + // RemoteDataTrackPublished) that may be delivered during or immediately after + // Connect() are not dropped. + auto delegate = std::make_unique(*this); + assert(delegate != nullptr); auto room = std::make_unique(); assert(room != nullptr); + room->setDelegate(delegate.get()); bool result = room->Connect(url, token, options); if (!result) { @@ -106,14 +128,8 @@ bool LiveKitBridge::connect(const std::string &url, const std::string &token, return false; } - // ---- Phase 3: commit and attach delegate under lock ---- - // Setting the delegate here (after Connect) ensures that any queued - // onTrackSubscribed events are delivered only after - // room_/delegate_/connected_ are all in a consistent state. - - auto delegate = std::make_unique(*this); - assert(delegate != nullptr); - room->setDelegate(delegate.get()); + // ---- Phase 3: commit under lock ---- + // room_/delegate_/connected_ are now in a consistent state. { std::lock_guard lock(mutex_); room_ = std::move(room); @@ -141,18 +157,21 @@ void LiveKitBridge::disconnect() { connecting_ = false; // Release all published tracks while the room/participant are still alive. - // This calls unpublishTrack() on each, ensuring participant_ is valid. for (auto &track : published_audio_tracks_) { track->release(); } for (auto &track : published_video_tracks_) { track->release(); } + for (auto &track : published_data_tracks_) { + track->release(); + } published_audio_tracks_.clear(); published_video_tracks_.clear(); + published_data_tracks_.clear(); // Close all streams (unblocks read loops) and collect threads - for (auto &[key, reader] : active_readers_) { + for (auto &[key, reader] : active_av_readers_) { if (reader.audio_stream) { reader.audio_stream->close(); } @@ -163,11 +182,23 @@ void LiveKitBridge::disconnect() { threads_to_join.emplace_back(std::move(reader.thread)); } } - active_readers_.clear(); + active_av_readers_.clear(); + + for (auto &[key, reader] : active_data_readers_) { + if (reader.subscription) { + reader.subscription->close(); + } + if (reader.thread.joinable()) { + threads_to_join.emplace_back(std::move(reader.thread)); + } + } + active_data_readers_.clear(); - // Clear callback registrations + // Clear callback registrations and pending data tracks audio_callbacks_.clear(); video_callbacks_.clear(); + data_callbacks_.clear(); + pending_remote_data_tracks_.clear(); // Tear down the room if (room_) { @@ -267,6 +298,26 @@ LiveKitBridge::createVideoTrack(const std::string &name, int width, int height, return bridge_track; } +std::shared_ptr +LiveKitBridge::createDataTrack(const std::string &name) { + std::lock_guard lock(mutex_); + + if (!connected_ || !room_) { + throw std::runtime_error( + "LiveKitBridge::createDataTrack: not connected to a room"); + } + + LK_LOG_INFO("[LiveKitBridge] Publishing data track \"{}\"...", name); + auto track = room_->localParticipant()->publishDataTrack(name); + LK_LOG_INFO("[LiveKitBridge] Data track \"{}\" published (sid={}).", name, + track->info().sid); + + auto bridge_track = std::shared_ptr( + new BridgeDataTrack(name, std::move(track), room_->localParticipant())); + published_data_tracks_.emplace_back(bridge_track); + return bridge_track; +} + // --------------------------------------------------------------- // Incoming frame callbacks // --------------------------------------------------------------- @@ -300,6 +351,35 @@ void LiveKitBridge::setOnVideoFrameCallback( video_callbacks_[key] = std::move(callback); } +void LiveKitBridge::setOnDataFrameCallback( + const std::string &participant_identity, const std::string &track_name, + DataFrameCallback callback) { + std::thread old_thread; + { + std::lock_guard lock(mutex_); + LK_LOG_INFO( + "[LiveKitBridge] Registered data callback for (\"{}\", \"{}\").", + participant_identity, track_name); + DataCallbackKey key{participant_identity, track_name}; + data_callbacks_[key] = std::move(callback); + + // If this track was already published and stored as pending, start the + // reader now (mirrors late registration for audio/video). + auto pending_it = pending_remote_data_tracks_.find(key); + if (pending_it != pending_remote_data_tracks_.end()) { + auto track = std::move(pending_it->second); + pending_remote_data_tracks_.erase(pending_it); + auto cb_it = data_callbacks_.find(key); + if (cb_it != data_callbacks_.end()) { + old_thread = startDataReader(key, track, cb_it->second); + } + } + } + if (old_thread.joinable()) { + old_thread.join(); + } +} + void LiveKitBridge::clearOnAudioFrameCallback( const std::string &participant_identity, livekit::TrackSource source) { std::thread thread_to_join; @@ -328,6 +408,20 @@ void LiveKitBridge::clearOnVideoFrameCallback( } } +void LiveKitBridge::clearOnDataFrameCallback( + const std::string &participant_identity, const std::string &track_name) { + std::thread thread_to_join; + { + std::lock_guard lock(mutex_); + DataCallbackKey key{participant_identity, track_name}; + data_callbacks_.erase(key); + thread_to_join = extractDataReaderThread(key); + } + if (thread_to_join.joinable()) { + thread_to_join.join(); + } +} + // --------------------------------------------------------------- // Internal: track subscribe / unsubscribe from delegate // --------------------------------------------------------------- @@ -351,6 +445,10 @@ void LiveKitBridge::onTrackSubscribed( if (it != video_callbacks_.end()) { old_thread = startVideoReader(key, track, it->second); } + } else { + LK_LOG_INFO( + "[LiveKitBridge] Track subscribed: \"{}\" from \"{}\" (sid={})", + track->name(), participant_identity, track->sid()); } } // If this key already had a reader (e.g. track was re-subscribed), the old @@ -375,6 +473,43 @@ void LiveKitBridge::onTrackUnsubscribed(const std::string &participant_identity, } } +void LiveKitBridge::onRemoteDataTrackPublished( + std::shared_ptr track) { + LK_LOG_INFO("[LiveKitBridge] Remote data track published: \"{}\" from \"{}\" " + "(sid={})", + track->info().name, track->publisherIdentity(), + track->info().sid); + + std::thread old_thread; + std::string identity; + std::string track_name; + + { + std::lock_guard lock(mutex_); + + DataCallbackKey key{track->publisherIdentity(), track->info().name}; + identity = key.identity; + track_name = key.track_name; + + auto it = data_callbacks_.find(key); + if (it != data_callbacks_.end()) { + LK_LOG_INFO("[LiveKitBridge] Found matching callback for ({}, {}), " + "starting data reader.", + key.identity, key.track_name); + old_thread = startDataReader(key, track, it->second); + } else { + LK_LOG_INFO("[LiveKitBridge] No callback registered yet for ({}, {}); " + "storing as pending (will start when callback is set).", + key.identity, key.track_name); + pending_remote_data_tracks_[key] = track; + } + } + + if (old_thread.joinable()) { + old_thread.join(); + } +} + // --------------------------------------------------------------- // Internal: reader thread management // --------------------------------------------------------------- @@ -382,8 +517,8 @@ void LiveKitBridge::onTrackUnsubscribed(const std::string &participant_identity, std::thread LiveKitBridge::extractReaderThread(const CallbackKey &key) { // Caller must hold mutex_. // Closes the stream and extracts the thread for the caller to join. - auto it = active_readers_.find(key); - if (it == active_readers_.end()) { + auto it = active_av_readers_.find(key); + if (it == active_av_readers_.end()) { return {}; } @@ -398,7 +533,7 @@ std::thread LiveKitBridge::extractReaderThread(const CallbackKey &key) { } auto thread = std::move(reader.thread); - active_readers_.erase(it); + active_av_readers_.erase(it); return thread; } @@ -434,10 +569,10 @@ LiveKitBridge::startAudioReader(const CallbackKey &key, } }); - active_readers_[key] = std::move(reader); - if (active_readers_.size() > kMaxActiveReaders) { - LK_LOG_WARN("More than expected active readers. Need to evaluate how much " - "to expect/support."); + active_av_readers_[key] = std::move(reader); + if (active_av_readers_.size() > kMaxActiveReaders) { + LK_LOG_ERROR("[LiveKitBridge] More than expected active readers. Need to " + "evaluate how much to expect/support."); } return old_thread; } @@ -475,12 +610,79 @@ LiveKitBridge::startVideoReader(const CallbackKey &key, } }); - active_readers_[key] = std::move(reader); - if (active_readers_.size() > kMaxActiveReaders) { - LK_LOG_WARN("More than expected active readers. Need to evaluate how much " - "to expect/support."); + active_av_readers_[key] = std::move(reader); + if (active_av_readers_.size() > kMaxActiveReaders) { + LK_LOG_ERROR("[LiveKitBridge] More than expected active readers. Need to " + "evaluate how much to expect/support."); } return old_thread; } +std::thread LiveKitBridge::startDataReader( + const DataCallbackKey &key, + const std::shared_ptr &track, + DataFrameCallback cb) { + auto old_thread = extractDataReaderThread(key); + + LK_LOG_INFO("[LiveKitBridge] Subscribing to data track \"{}\" from \"{}\"...", + key.track_name, key.identity); + + std::shared_ptr subscription; + try { + subscription = track->subscribe(); + } catch (const std::exception &e) { + LK_LOG_ERROR("[LiveKitBridge] Failed to subscribe to data track \"{}\" " + "from \"{}\": {}", + key.track_name, key.identity, e.what()); + return old_thread; + } + + LK_LOG_INFO("[LiveKitBridge] Subscribed to data track \"{}\"; starting " + "reader thread.", + key.track_name); + + auto sub_copy = subscription; + auto track_name = key.track_name; + auto identity = key.identity; + + ActiveDataReader reader; + reader.remote_track = track; + reader.subscription = std::move(subscription); + reader.thread = std::thread([sub_copy, cb, track_name, identity]() { + LK_LOG_INFO( + "[LiveKitBridge] Data reader thread running for \"{}\" from \"{}\".", + track_name, identity); + livekit::DataFrame frame; + while (sub_copy->read(frame)) { + try { + cb(frame.payload, frame.user_timestamp); + } catch (const std::exception &e) { + LK_LOG_ERROR("[LiveKitBridge] Data callback exception: {}", e.what()); + } + } + LK_LOG_INFO( + "[LiveKitBridge] Data reader thread exiting for \"{}\" from \"{}\".", + track_name, identity); + }); + + active_data_readers_[key] = std::move(reader); + return old_thread; +} + +std::thread LiveKitBridge::extractDataReaderThread(const DataCallbackKey &key) { + auto it = active_data_readers_.find(key); + if (it == active_data_readers_.end()) { + return {}; + } + + auto &reader = it->second; + + if (reader.subscription) { + reader.subscription->close(); + } + + auto thread = std::move(reader.thread); + active_data_readers_.erase(it); + return thread; +} } // namespace livekit_bridge diff --git a/bridge/tests/CMakeLists.txt b/bridge/tests/CMakeLists.txt index 258681e..f5114cf 100644 --- a/bridge/tests/CMakeLists.txt +++ b/bridge/tests/CMakeLists.txt @@ -25,70 +25,154 @@ enable_testing() include(GoogleTest) # ============================================================================ -# Bridge Unit Tests +# Helper: copy shared libraries to a test executable's output directory # ============================================================================ - -file(GLOB BRIDGE_TEST_SOURCES - "${CMAKE_CURRENT_SOURCE_DIR}/*.cpp" -) - -if(BRIDGE_TEST_SOURCES) - add_executable(livekit_bridge_tests - ${BRIDGE_TEST_SOURCES} - ) - - target_link_libraries(livekit_bridge_tests - PRIVATE - livekit_bridge - GTest::gtest_main - ) - - # Copy shared libraries to test executable directory +function(bridge_test_copy_libs target) if(WIN32) - add_custom_command(TARGET livekit_bridge_tests POST_BUILD + add_custom_command(TARGET ${target} POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different $ - $ + $ COMMAND ${CMAKE_COMMAND} -E copy_if_different $ - $ + $ COMMAND ${CMAKE_COMMAND} -E copy_if_different "$/livekit_ffi.dll" - $ - COMMENT "Copying DLLs to bridge test directory" + $ + COMMENT "Copying DLLs to ${target} directory" ) elseif(APPLE) - add_custom_command(TARGET livekit_bridge_tests POST_BUILD + add_custom_command(TARGET ${target} POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different $ - $ + $ COMMAND ${CMAKE_COMMAND} -E copy_if_different $ - $ + $ COMMAND ${CMAKE_COMMAND} -E copy_if_different "$/liblivekit_ffi.dylib" - $ - COMMENT "Copying dylibs to bridge test directory" + $ + COMMENT "Copying dylibs to ${target} directory" ) else() - add_custom_command(TARGET livekit_bridge_tests POST_BUILD + add_custom_command(TARGET ${target} POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different $ - $ + $ COMMAND ${CMAKE_COMMAND} -E copy_if_different $ - $ + $ COMMAND ${CMAKE_COMMAND} -E copy_if_different "$/liblivekit_ffi.so" - $ - COMMENT "Copying shared libraries to bridge test directory" + $ + COMMENT "Copying shared libraries to ${target} directory" ) endif() +endfunction() + +# ============================================================================ +# Bridge Unit Tests (existing — files in tests/ root) +# ============================================================================ + +file(GLOB BRIDGE_TEST_SOURCES + "${CMAKE_CURRENT_SOURCE_DIR}/unit/*.cpp" +) + +if(BRIDGE_TEST_SOURCES) + add_executable(livekit_bridge_unit_tests + ${BRIDGE_TEST_SOURCES} + ) + + target_link_libraries(livekit_bridge_unit_tests + PRIVATE + livekit_bridge + GTest::gtest_main + ) + + bridge_test_copy_libs(livekit_bridge_unit_tests) + + gtest_discover_tests(livekit_bridge_unit_tests + WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} + PROPERTIES + LABELS "bridge_unit_tests" + ) +endif() + +# ============================================================================ +# Bridge Integration Tests (tests/integration/) +# ============================================================================ + +file(GLOB BRIDGE_INTEGRATION_SOURCES + "${CMAKE_CURRENT_SOURCE_DIR}/integration/*.cpp" +) + +if(BRIDGE_INTEGRATION_SOURCES) + add_executable(livekit_bridge_integration_tests + ${BRIDGE_INTEGRATION_SOURCES} + ) + + target_link_libraries(livekit_bridge_integration_tests + PRIVATE + livekit_bridge + GTest::gtest_main + ) + + bridge_test_copy_libs(livekit_bridge_integration_tests) + + gtest_discover_tests(livekit_bridge_integration_tests + WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} + PROPERTIES + LABELS "bridge_integration" + ) +endif() + +# ============================================================================ +# Bridge Stress Tests (tests/stress/) +# ============================================================================ + +file(GLOB BRIDGE_STRESS_SOURCES + "${CMAKE_CURRENT_SOURCE_DIR}/stress/*.cpp" +) - # Register tests with CTest - gtest_discover_tests(livekit_bridge_tests +if(BRIDGE_STRESS_SOURCES) + add_executable(livekit_bridge_stress_tests + ${BRIDGE_STRESS_SOURCES} + ) + + target_link_libraries(livekit_bridge_stress_tests + PRIVATE + livekit_bridge + GTest::gtest_main + ) + + bridge_test_copy_libs(livekit_bridge_stress_tests) + + gtest_discover_tests(livekit_bridge_stress_tests WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} PROPERTIES - LABELS "bridge_unit" + LABELS "bridge_stress" + TIMEOUT 300 ) endif() + +# ============================================================================ +# Combined target +# ============================================================================ + +add_custom_target(run_bridge_tests + COMMAND ${CMAKE_CTEST_COMMAND} -L "bridge_unit|bridge_integration|bridge_stress" --output-on-failure + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + COMMENT "Running all bridge tests" +) + +if(TARGET livekit_bridge_unit_tests) + add_dependencies(run_bridge_tests livekit_bridge_unit_tests) +endif() + +if(TARGET livekit_bridge_integration_tests) + add_dependencies(run_bridge_tests livekit_bridge_integration_tests) +endif() + +if(TARGET livekit_bridge_stress_tests) + add_dependencies(run_bridge_tests livekit_bridge_stress_tests) +endif() diff --git a/bridge/tests/common/bridge_test_common.h b/bridge/tests/common/bridge_test_common.h new file mode 100644 index 0000000..9389a50 --- /dev/null +++ b/bridge/tests/common/bridge_test_common.h @@ -0,0 +1,311 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace livekit_bridge { +namespace test { + +using namespace std::chrono_literals; + +constexpr int kDefaultTestIterations = 10; +constexpr int kDefaultStressDurationSeconds = 600; + +/** + * Test configuration loaded from the same environment variables used by the + * base SDK tests (see README "RPC Test Environment Variables"). + * + * LIVEKIT_URL - WebSocket URL of the LiveKit server + * LIVEKIT_CALLER_TOKEN - Token for the caller/sender participant + * LIVEKIT_RECEIVER_TOKEN - Token for the receiver participant + * TEST_ITERATIONS - Number of iterations (default: 10) + * STRESS_DURATION_SECONDS - Duration for stress tests (default: 600) + */ +struct BridgeTestConfig { + std::string url; + std::string caller_token; + std::string receiver_token; + int test_iterations; + int stress_duration_seconds; + bool available = false; + + static BridgeTestConfig fromEnv() { + BridgeTestConfig config; + const char *url = std::getenv("LIVEKIT_URL"); + const char *caller_token = std::getenv("LIVEKIT_CALLER_TOKEN"); + const char *receiver_token = std::getenv("LIVEKIT_RECEIVER_TOKEN"); + const char *iterations_env = std::getenv("TEST_ITERATIONS"); + const char *duration_env = std::getenv("STRESS_DURATION_SECONDS"); + + if (url && caller_token && receiver_token) { + config.url = url; + config.caller_token = caller_token; + config.receiver_token = receiver_token; + config.available = true; + } + + config.test_iterations = + iterations_env ? std::atoi(iterations_env) : kDefaultTestIterations; + config.stress_duration_seconds = + duration_env ? std::atoi(duration_env) : kDefaultStressDurationSeconds; + + return config; + } +}; + +/** + * Thread-safe latency statistics collector. + * Identical to livekit::test::LatencyStats but lives in the bridge namespace + * to avoid linking against the base SDK test helpers. + */ +class LatencyStats { +public: + void addMeasurement(double latency_ms) { + std::lock_guard lock(mutex_); + measurements_.push_back(latency_ms); + } + + void printStats(const std::string &title) const { + std::lock_guard lock(mutex_); + + if (measurements_.empty()) { + std::cout << "\n" << title << ": No measurements collected" << std::endl; + return; + } + + std::vector sorted = measurements_; + std::sort(sorted.begin(), sorted.end()); + + double sum = std::accumulate(sorted.begin(), sorted.end(), 0.0); + double avg = sum / sorted.size(); + + std::cout << "\n========================================" << std::endl; + std::cout << " " << title << std::endl; + std::cout << "========================================" << std::endl; + std::cout << "Samples: " << sorted.size() << std::endl; + std::cout << std::fixed << std::setprecision(2); + std::cout << "Min: " << sorted.front() << " ms" << std::endl; + std::cout << "Avg: " << avg << " ms" << std::endl; + std::cout << "P50: " << getPercentile(sorted, 50) << " ms" + << std::endl; + std::cout << "P95: " << getPercentile(sorted, 95) << " ms" + << std::endl; + std::cout << "P99: " << getPercentile(sorted, 99) << " ms" + << std::endl; + std::cout << "Max: " << sorted.back() << " ms" << std::endl; + std::cout << "========================================\n" << std::endl; + } + + size_t count() const { + std::lock_guard lock(mutex_); + return measurements_.size(); + } + + void clear() { + std::lock_guard lock(mutex_); + measurements_.clear(); + } + +private: + static double getPercentile(const std::vector &sorted, + int percentile) { + if (sorted.empty()) + return 0.0; + size_t index = (sorted.size() * percentile) / 100; + if (index >= sorted.size()) + index = sorted.size() - 1; + return sorted[index]; + } + + mutable std::mutex mutex_; + std::vector measurements_; +}; + +/** + * Extended statistics collector for stress tests. + */ +class StressTestStats { +public: + void recordCall(bool success, double latency_ms, size_t payload_size = 0) { + std::lock_guard lock(mutex_); + total_calls_++; + if (success) { + successful_calls_++; + latencies_.push_back(latency_ms); + total_bytes_ += payload_size; + } else { + failed_calls_++; + } + } + + void recordError(const std::string &error_type) { + std::lock_guard lock(mutex_); + error_counts_[error_type]++; + } + + void printStats(const std::string &title = "Stress Test Statistics") const { + std::lock_guard lock(mutex_); + + std::cout << "\n========================================" << std::endl; + std::cout << " " << title << std::endl; + std::cout << "========================================" << std::endl; + std::cout << "Total calls: " << total_calls_ << std::endl; + std::cout << "Successful: " << successful_calls_ << std::endl; + std::cout << "Failed: " << failed_calls_ << std::endl; + std::cout << "Success rate: " << std::fixed << std::setprecision(2) + << (total_calls_ > 0 + ? (100.0 * successful_calls_ / total_calls_) + : 0.0) + << "%" << std::endl; + std::cout << "Total bytes: " << total_bytes_ << " (" + << (total_bytes_ / (1024.0 * 1024.0)) << " MB)" << std::endl; + + if (!latencies_.empty()) { + std::vector sorted = latencies_; + std::sort(sorted.begin(), sorted.end()); + + double sum = + std::accumulate(sorted.begin(), sorted.end(), 0.0); + double avg = sum / sorted.size(); + + std::cout << "\nLatency (ms):" << std::endl; + std::cout << " Min: " << sorted.front() << std::endl; + std::cout << " Avg: " << avg << std::endl; + std::cout << " P50: " + << sorted[sorted.size() * 50 / 100] << std::endl; + std::cout << " P95: " + << sorted[sorted.size() * 95 / 100] << std::endl; + std::cout << " P99: " + << sorted[sorted.size() * 99 / 100] << std::endl; + std::cout << " Max: " << sorted.back() << std::endl; + } + + if (!error_counts_.empty()) { + std::cout << "\nError breakdown:" << std::endl; + for (const auto &pair : error_counts_) { + std::cout << " " << pair.first << ": " << pair.second << std::endl; + } + } + + std::cout << "========================================\n" << std::endl; + } + + int totalCalls() const { + std::lock_guard lock(mutex_); + return total_calls_; + } + + int successfulCalls() const { + std::lock_guard lock(mutex_); + return successful_calls_; + } + + int failedCalls() const { + std::lock_guard lock(mutex_); + return failed_calls_; + } + +private: + mutable std::mutex mutex_; + int total_calls_ = 0; + int successful_calls_ = 0; + int failed_calls_ = 0; + size_t total_bytes_ = 0; + std::vector latencies_; + std::map error_counts_; +}; + +/** + * Base test fixture for bridge E2E tests. + * + * IMPORTANT — SDK lifecycle constraints: + * + * • livekit::initialize() / livekit::shutdown() operate on a process-global + * singleton (FfiClient). shutdown() calls livekit_ffi_dispose() which + * tears down the Rust runtime. Re-initializing after a dispose can leave + * internal Rust state corrupt when done many times in rapid succession. + * + * • Each LiveKitBridge instance independently calls initialize()/shutdown() + * in connect()/disconnect(). With two bridges in the same test the first + * one to disconnect() shuts down the SDK while the second is still alive. + * + * Our strategy: + * 1. Tests should let bridge destructors handle disconnect. Do NOT call + * bridge.disconnect() at the end of a test — just let the bridge go + * out of scope and its destructor will disconnect and (eventually) + * call shutdown. + * 2. If you need to explicitly disconnect mid-test (e.g. to test + * lifecycle), accept that this triggers a shutdown. Add a 1 s sleep + * after destruction before creating the next bridge so the Rust + * runtime fully cleans up. + */ +class BridgeTestBase : public ::testing::Test { +protected: + void SetUp() override { config_ = BridgeTestConfig::fromEnv(); } + + void skipIfNotConfigured() { + if (!config_.available) { + GTEST_SKIP() << "LIVEKIT_URL, LIVEKIT_CALLER_TOKEN, and " + "LIVEKIT_RECEIVER_TOKEN not set"; + } + } + + /** + * Connect two bridges (caller and receiver) and verify both are connected. + * Returns false if either connection fails. + */ + bool connectPair(LiveKitBridge &caller, LiveKitBridge &receiver) { + livekit::RoomOptions options; + options.auto_subscribe = true; + + bool receiver_ok = + receiver.connect(config_.url, config_.receiver_token, options); + if (!receiver_ok) + return false; + + bool caller_ok = + caller.connect(config_.url, config_.caller_token, options); + if (!caller_ok) { + receiver.disconnect(); + return false; + } + + // Allow time for peer discovery + std::this_thread::sleep_for(2s); + return true; + } + + BridgeTestConfig config_; +}; + +} // namespace test +} // namespace livekit_bridge diff --git a/bridge/tests/integration/test_bridge_audio_roundtrip.cpp b/bridge/tests/integration/test_bridge_audio_roundtrip.cpp new file mode 100644 index 0000000..bad61bb --- /dev/null +++ b/bridge/tests/integration/test_bridge_audio_roundtrip.cpp @@ -0,0 +1,321 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../common/bridge_test_common.h" +#include +#include + +namespace livekit_bridge { +namespace test { + +constexpr int kAudioSampleRate = 48000; +constexpr int kAudioChannels = 1; +constexpr int kAudioFrameDurationMs = 10; +constexpr int kSamplesPerFrame = + kAudioSampleRate * kAudioFrameDurationMs / 1000; +constexpr double kHighEnergyThreshold = 0.3; +constexpr int kHighEnergyFramesPerPulse = 5; + +static std::vector generateHighEnergyFrame(int samples) { + std::vector data(samples * kAudioChannels); + const double frequency = 1000.0; + const double amplitude = 30000.0; + for (int i = 0; i < samples; ++i) { + double t = static_cast(i) / kAudioSampleRate; + auto sample = static_cast( + amplitude * std::sin(2.0 * M_PI * frequency * t)); + for (int ch = 0; ch < kAudioChannels; ++ch) { + data[i * kAudioChannels + ch] = sample; + } + } + return data; +} + +static std::vector generateSilentFrame(int samples) { + return std::vector(samples * kAudioChannels, 0); +} + +static double calculateEnergy(const std::vector &samples) { + if (samples.empty()) + return 0.0; + double sum_squared = 0.0; + for (auto s : samples) { + double normalized = static_cast(s) / 32768.0; + sum_squared += normalized * normalized; + } + return std::sqrt(sum_squared / samples.size()); +} + +class BridgeAudioRoundtripTest : public BridgeTestBase {}; + +// --------------------------------------------------------------------------- +// Test 1: Basic audio frame round-trip through the bridge API. +// +// Caller bridge publishes an audio track, receiver bridge receives frames via +// setOnAudioFrameCallback. We send high-energy pulses interleaved with +// silence and verify that the receiver detects them. +// --------------------------------------------------------------------------- +TEST_F(BridgeAudioRoundtripTest, AudioFrameRoundTrip) { + skipIfNotConfigured(); + + std::cout << "\n=== Bridge Audio Frame Round-Trip Test ===" << std::endl; + + LiveKitBridge caller; + LiveKitBridge receiver; + + ASSERT_TRUE(connectPair(caller, receiver)) + << "Failed to connect caller/receiver pair"; + + std::cout << "Both bridges connected." << std::endl; + + auto audio_track = caller.createAudioTrack( + "roundtrip-mic", kAudioSampleRate, kAudioChannels, + livekit::TrackSource::SOURCE_MICROPHONE); + ASSERT_NE(audio_track, nullptr); + + std::cout << "Audio track published." << std::endl; + + std::atomic frames_received{0}; + std::atomic high_energy_frames{0}; + + const std::string caller_identity = "rpc-caller"; + + receiver.setOnAudioFrameCallback( + caller_identity, livekit::TrackSource::SOURCE_MICROPHONE, + [&](const livekit::AudioFrame &frame) { + frames_received++; + double energy = calculateEnergy(frame.data()); + if (energy > kHighEnergyThreshold) { + high_energy_frames++; + } + }); + + std::cout << "Callback registered, sending audio..." << std::endl; + + const int total_frames = 500; + const int frames_between_pulses = 100; + auto next_frame_time = std::chrono::steady_clock::now(); + const auto frame_duration = std::chrono::milliseconds(kAudioFrameDurationMs); + int pulses_sent = 0; + int high_energy_remaining = 0; + + for (int i = 0; i < total_frames; ++i) { + std::this_thread::sleep_until(next_frame_time); + next_frame_time += frame_duration; + + std::vector frame_data; + + if (high_energy_remaining > 0) { + frame_data = generateHighEnergyFrame(kSamplesPerFrame); + high_energy_remaining--; + } else if (i > 0 && i % frames_between_pulses == 0) { + frame_data = generateHighEnergyFrame(kSamplesPerFrame); + high_energy_remaining = kHighEnergyFramesPerPulse - 1; + pulses_sent++; + std::cout << " Sent pulse " << pulses_sent << std::endl; + } else { + frame_data = generateSilentFrame(kSamplesPerFrame); + } + + audio_track->pushFrame(frame_data, kSamplesPerFrame); + } + + std::this_thread::sleep_for(2s); + + std::cout << "\nResults:" << std::endl; + std::cout << " Pulses sent: " << pulses_sent << std::endl; + std::cout << " Frames received: " << frames_received.load() + << std::endl; + std::cout << " High-energy frames rx: " << high_energy_frames.load() + << std::endl; + + EXPECT_GT(frames_received.load(), 0) + << "Receiver should have received at least one audio frame"; + EXPECT_GT(high_energy_frames.load(), 0) + << "Receiver should have detected at least one high-energy pulse"; + + // Clear callback before bridges go out of scope so the reader thread + // is joined while the atomic counters above are still alive. + receiver.clearOnAudioFrameCallback( + caller_identity, livekit::TrackSource::SOURCE_MICROPHONE); +} + +// --------------------------------------------------------------------------- +// Test 2: Audio latency measurement through the bridge. +// +// Same energy-detection approach as the base SDK's AudioLatency test but +// exercises the full bridge pipeline: pushFrame() → SFU → reader thread → +// AudioFrameCallback. +// --------------------------------------------------------------------------- +TEST_F(BridgeAudioRoundtripTest, AudioLatencyMeasurement) { + skipIfNotConfigured(); + + std::cout << "\n=== Bridge Audio Latency Measurement ===" << std::endl; + + LiveKitBridge caller; + LiveKitBridge receiver; + + ASSERT_TRUE(connectPair(caller, receiver)); + + auto audio_track = caller.createAudioTrack( + "latency-mic", kAudioSampleRate, kAudioChannels, + livekit::TrackSource::SOURCE_MICROPHONE); + ASSERT_NE(audio_track, nullptr); + + LatencyStats stats; + std::atomic running{true}; + std::atomic last_send_time_us{0}; + std::atomic waiting_for_echo{false}; + std::atomic missed_pulses{0}; + constexpr uint64_t kEchoTimeoutUs = 2'000'000; + + const std::string caller_identity = "rpc-caller"; + + receiver.setOnAudioFrameCallback( + caller_identity, livekit::TrackSource::SOURCE_MICROPHONE, + [&](const livekit::AudioFrame &frame) { + double energy = calculateEnergy(frame.data()); + if (waiting_for_echo.load() && energy > kHighEnergyThreshold) { + uint64_t rx_us = + std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); + uint64_t tx_us = last_send_time_us.load(); + if (tx_us > 0) { + double latency_ms = (rx_us - tx_us) / 1000.0; + if (latency_ms > 0 && latency_ms < 5000) { + stats.addMeasurement(latency_ms); + std::cout << " Latency: " << std::fixed << std::setprecision(2) + << latency_ms << " ms" << std::endl; + } + waiting_for_echo.store(false); + } + } + }); + + const int total_pulses = 10; + const int frames_between_pulses = 100; + int pulses_sent = 0; + int high_energy_remaining = 0; + uint64_t pulse_send_time = 0; + + auto next_frame_time = std::chrono::steady_clock::now(); + const auto frame_duration = std::chrono::milliseconds(kAudioFrameDurationMs); + int frame_count = 0; + + while (running.load() && pulses_sent < total_pulses) { + std::this_thread::sleep_until(next_frame_time); + next_frame_time += frame_duration; + + if (waiting_for_echo.load() && pulse_send_time > 0) { + uint64_t now_us = + std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); + if (now_us - pulse_send_time > kEchoTimeoutUs) { + std::cout << " Echo timeout for pulse " << pulses_sent << std::endl; + waiting_for_echo.store(false); + missed_pulses++; + pulse_send_time = 0; + high_energy_remaining = 0; + } + } + + std::vector frame_data; + + if (high_energy_remaining > 0) { + frame_data = generateHighEnergyFrame(kSamplesPerFrame); + high_energy_remaining--; + } else if (frame_count > 0 && + frame_count % frames_between_pulses == 0 && + !waiting_for_echo.load()) { + frame_data = generateHighEnergyFrame(kSamplesPerFrame); + high_energy_remaining = kHighEnergyFramesPerPulse - 1; + + pulse_send_time = + std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); + last_send_time_us.store(pulse_send_time); + waiting_for_echo.store(true); + pulses_sent++; + std::cout << "Sent pulse " << pulses_sent << "/" << total_pulses + << std::endl; + } else { + frame_data = generateSilentFrame(kSamplesPerFrame); + } + + audio_track->pushFrame(frame_data, kSamplesPerFrame); + frame_count++; + } + + std::this_thread::sleep_for(2s); + + stats.printStats("Bridge Audio Latency Statistics"); + + if (missed_pulses > 0) { + std::cout << "Missed pulses (timeout): " << missed_pulses << std::endl; + } + + EXPECT_GT(stats.count(), 0u) + << "At least one audio latency measurement should be recorded"; + + receiver.clearOnAudioFrameCallback( + caller_identity, livekit::TrackSource::SOURCE_MICROPHONE); +} + +// --------------------------------------------------------------------------- +// Test 3: Connect → publish → disconnect cycle for audio tracks. +// +// Each cycle creates a bridge in a scoped block so the destructor runs +// before the next cycle begins. A sleep between cycles gives the Rust +// runtime time to fully tear down. +// --------------------------------------------------------------------------- +TEST_F(BridgeAudioRoundtripTest, ConnectPublishDisconnectCycle) { + skipIfNotConfigured(); + + const int cycles = config_.test_iterations; + std::cout << "\n=== Bridge Audio Connect/Disconnect Cycles ===" << std::endl; + std::cout << "Cycles: " << cycles << std::endl; + + for (int i = 0; i < cycles; ++i) { + { + LiveKitBridge bridge; + livekit::RoomOptions options; + options.auto_subscribe = true; + + bool connected = + bridge.connect(config_.url, config_.caller_token, options); + ASSERT_TRUE(connected) << "Cycle " << i << ": connect failed"; + + auto track = bridge.createAudioTrack( + "cycle-mic", kAudioSampleRate, kAudioChannels, + livekit::TrackSource::SOURCE_MICROPHONE); + ASSERT_NE(track, nullptr); + + for (int f = 0; f < 10; ++f) { + auto data = generateSilentFrame(kSamplesPerFrame); + track->pushFrame(data, kSamplesPerFrame); + } + } // bridge destroyed here → disconnect + shutdown + + std::cout << " Cycle " << (i + 1) << "/" << cycles << " OK" << std::endl; + std::this_thread::sleep_for(1s); + } +} + +} // namespace test +} // namespace livekit_bridge diff --git a/bridge/tests/integration/test_bridge_data_roundtrip.cpp b/bridge/tests/integration/test_bridge_data_roundtrip.cpp new file mode 100644 index 0000000..70c1899 --- /dev/null +++ b/bridge/tests/integration/test_bridge_data_roundtrip.cpp @@ -0,0 +1,295 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../common/bridge_test_common.h" +#include +#include + +namespace livekit_bridge { +namespace test { + +static std::vector generatePayload(size_t size) { + static thread_local std::mt19937 gen(std::random_device{}()); + std::uniform_int_distribution dist(0, 255); + std::vector data(size); + for (auto &b : data) { + b = static_cast(dist(gen)); + } + return data; +} + +class BridgeDataRoundtripTest : public BridgeTestBase {}; + +// --------------------------------------------------------------------------- +// Test 1: Basic data track round-trip. +// +// Caller publishes a data track, receiver registers a callback, caller sends +// frames, receiver verifies payload integrity. +// --------------------------------------------------------------------------- +TEST_F(BridgeDataRoundtripTest, DataFrameRoundTrip) { + skipIfNotConfigured(); + + std::cout << "\n=== Bridge Data Frame Round-Trip Test ===" << std::endl; + + LiveKitBridge caller; + LiveKitBridge receiver; + + ASSERT_TRUE(connectPair(caller, receiver)); + + const std::string track_name = "roundtrip-data"; + const std::string caller_identity = "rpc-caller"; + + auto data_track = caller.createDataTrack(track_name); + ASSERT_NE(data_track, nullptr); + ASSERT_TRUE(data_track->isPublished()); + + std::cout << "Data track published." << std::endl; + + std::mutex rx_mutex; + std::condition_variable rx_cv; + std::vector> received_payloads; + std::vector> received_timestamps; + + receiver.setOnDataFrameCallback( + caller_identity, track_name, + [&](const std::vector &payload, + std::optional user_timestamp) { + std::lock_guard lock(rx_mutex); + received_payloads.push_back(payload); + received_timestamps.push_back(user_timestamp); + rx_cv.notify_all(); + }); + + // Give the subscription time to be established + std::this_thread::sleep_for(3s); + + std::cout << "Sending data frames..." << std::endl; + + const int num_frames = 10; + std::vector> sent_payloads; + std::vector sent_timestamps; + + for (int i = 0; i < num_frames; ++i) { + auto payload = generatePayload(256); + auto ts = static_cast(i * 1000); + sent_payloads.push_back(payload); + sent_timestamps.push_back(ts); + + bool pushed = data_track->pushFrame(payload, ts); + EXPECT_TRUE(pushed) << "pushFrame failed for frame " << i; + + std::this_thread::sleep_for(100ms); + } + + { + std::unique_lock lock(rx_mutex); + rx_cv.wait_for(lock, 10s, [&] { + return received_payloads.size() >= static_cast(num_frames); + }); + } + + std::cout << "\nResults:" << std::endl; + std::cout << " Frames sent: " << num_frames << std::endl; + std::cout << " Frames received: " << received_payloads.size() << std::endl; + + EXPECT_EQ(received_payloads.size(), static_cast(num_frames)) + << "Should receive all sent frames"; + + for (size_t i = 0; + i < std::min(received_payloads.size(), sent_payloads.size()); ++i) { + EXPECT_EQ(received_payloads[i], sent_payloads[i]) + << "Payload mismatch at frame " << i; + ASSERT_TRUE(received_timestamps[i].has_value()) + << "Missing timestamp at frame " << i; + EXPECT_EQ(received_timestamps[i].value(), sent_timestamps[i]) + << "Timestamp mismatch at frame " << i; + } + + receiver.clearOnDataFrameCallback(caller_identity, track_name); +} + +// --------------------------------------------------------------------------- +// Test 2: Data track with callback registered AFTER track is published. +// +// Exercises the bridge's pending_remote_data_tracks_ mechanism: the remote +// data track is published before the receiver registers its callback. +// --------------------------------------------------------------------------- +TEST_F(BridgeDataRoundtripTest, LateCallbackRegistration) { + skipIfNotConfigured(); + + std::cout << "\n=== Bridge Data Late Callback Registration Test ===" + << std::endl; + + LiveKitBridge caller; + LiveKitBridge receiver; + + ASSERT_TRUE(connectPair(caller, receiver)); + + const std::string track_name = "late-callback-data"; + const std::string caller_identity = "rpc-caller"; + + auto data_track = caller.createDataTrack(track_name); + ASSERT_NE(data_track, nullptr); + + std::cout << "Data track published, waiting before registering callback..." + << std::endl; + + std::this_thread::sleep_for(3s); + + std::atomic frames_received{0}; + std::condition_variable rx_cv; + std::mutex rx_mutex; + + receiver.setOnDataFrameCallback( + caller_identity, track_name, + [&](const std::vector &, + std::optional) { + frames_received++; + rx_cv.notify_all(); + }); + + std::cout << "Callback registered (late), sending frames..." << std::endl; + + std::this_thread::sleep_for(2s); + + const int num_frames = 5; + for (int i = 0; i < num_frames; ++i) { + auto payload = generatePayload(128); + data_track->pushFrame(payload); + std::this_thread::sleep_for(100ms); + } + + { + std::unique_lock lock(rx_mutex); + rx_cv.wait_for(lock, 10s, [&] { + return frames_received.load() >= num_frames; + }); + } + + std::cout << "Frames received: " << frames_received.load() << std::endl; + + EXPECT_EQ(frames_received.load(), num_frames) + << "Late callback should still receive all frames"; + + receiver.clearOnDataFrameCallback(caller_identity, track_name); +} + +// --------------------------------------------------------------------------- +// Test 3: Varying payload sizes. +// +// Tests data track with payloads from tiny (1 byte) to large (64KB) to +// verify the bridge handles different frame sizes correctly. +// --------------------------------------------------------------------------- +TEST_F(BridgeDataRoundtripTest, VaryingPayloadSizes) { + skipIfNotConfigured(); + + std::cout << "\n=== Bridge Data Varying Payload Sizes Test ===" << std::endl; + + LiveKitBridge caller; + LiveKitBridge receiver; + + ASSERT_TRUE(connectPair(caller, receiver)); + + const std::string track_name = "size-test-data"; + const std::string caller_identity = "rpc-caller"; + + auto data_track = caller.createDataTrack(track_name); + ASSERT_NE(data_track, nullptr); + + std::mutex rx_mutex; + std::condition_variable rx_cv; + std::vector received_sizes; + + receiver.setOnDataFrameCallback( + caller_identity, track_name, + [&](const std::vector &payload, + std::optional) { + std::lock_guard lock(rx_mutex); + received_sizes.push_back(payload.size()); + rx_cv.notify_all(); + }); + + std::this_thread::sleep_for(3s); + + std::vector test_sizes = {1, 10, 100, 1024, 4096, 16384, 65536}; + + std::cout << "Sending " << test_sizes.size() + << " frames with varying sizes..." << std::endl; + + for (size_t sz : test_sizes) { + auto payload = generatePayload(sz); + bool pushed = data_track->pushFrame(payload); + EXPECT_TRUE(pushed) << "pushFrame failed for size " << sz; + std::this_thread::sleep_for(200ms); + } + + { + std::unique_lock lock(rx_mutex); + rx_cv.wait_for(lock, 15s, [&] { + return received_sizes.size() >= test_sizes.size(); + }); + } + + std::cout << "Received " << received_sizes.size() << "/" + << test_sizes.size() << " frames." << std::endl; + + EXPECT_EQ(received_sizes.size(), test_sizes.size()); + + for (size_t i = 0; + i < std::min(received_sizes.size(), test_sizes.size()); ++i) { + EXPECT_EQ(received_sizes[i], test_sizes[i]) + << "Size mismatch at index " << i; + } + + receiver.clearOnDataFrameCallback(caller_identity, track_name); +} + +// --------------------------------------------------------------------------- +// Test 4: Connect → publish data track → disconnect cycle. +// --------------------------------------------------------------------------- +TEST_F(BridgeDataRoundtripTest, ConnectPublishDisconnectCycle) { + skipIfNotConfigured(); + + const int cycles = config_.test_iterations; + std::cout << "\n=== Bridge Data Connect/Disconnect Cycles ===" << std::endl; + std::cout << "Cycles: " << cycles << std::endl; + + for (int i = 0; i < cycles; ++i) { + { + LiveKitBridge bridge; + livekit::RoomOptions options; + options.auto_subscribe = true; + + bool connected = + bridge.connect(config_.url, config_.caller_token, options); + ASSERT_TRUE(connected) << "Cycle " << i << ": connect failed"; + + auto track = bridge.createDataTrack("cycle-data"); + ASSERT_NE(track, nullptr); + + for (int f = 0; f < 5; ++f) { + auto payload = generatePayload(256); + track->pushFrame(payload); + } + } // bridge destroyed here → disconnect + shutdown + + std::cout << " Cycle " << (i + 1) << "/" << cycles << " OK" << std::endl; + std::this_thread::sleep_for(1s); + } +} + +} // namespace test +} // namespace livekit_bridge diff --git a/bridge/tests/stress/test_bridge_audio_stress.cpp b/bridge/tests/stress/test_bridge_audio_stress.cpp new file mode 100644 index 0000000..88ba97f --- /dev/null +++ b/bridge/tests/stress/test_bridge_audio_stress.cpp @@ -0,0 +1,276 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../common/bridge_test_common.h" +#include + +namespace livekit_bridge { +namespace test { + +constexpr int kStressAudioSampleRate = 48000; +constexpr int kStressAudioChannels = 1; +constexpr int kStressFrameDurationMs = 10; +constexpr int kStressSamplesPerFrame = + kStressAudioSampleRate * kStressFrameDurationMs / 1000; + +static std::vector makeSineFrame(int samples, double freq, + int &phase) { + std::vector data(samples * kStressAudioChannels); + const double amplitude = 16000.0; + for (int i = 0; i < samples; ++i) { + double t = static_cast(phase++) / kStressAudioSampleRate; + auto sample = static_cast( + amplitude * std::sin(2.0 * M_PI * freq * t)); + for (int ch = 0; ch < kStressAudioChannels; ++ch) { + data[i * kStressAudioChannels + ch] = sample; + } + } + return data; +} + +class BridgeAudioStressTest : public BridgeTestBase {}; + +// --------------------------------------------------------------------------- +// Sustained audio pushing: sends audio at real-time pace for the configured +// stress duration and tracks frames received, delivery rate, and errors. +// --------------------------------------------------------------------------- +TEST_F(BridgeAudioStressTest, SustainedAudioPush) { + skipIfNotConfigured(); + + std::cout << "\n=== Bridge Sustained Audio Stress Test ===" << std::endl; + std::cout << "Duration: " << config_.stress_duration_seconds << " seconds" + << std::endl; + + LiveKitBridge caller; + LiveKitBridge receiver; + + ASSERT_TRUE(connectPair(caller, receiver)); + + auto audio_track = caller.createAudioTrack( + "stress-mic", kStressAudioSampleRate, kStressAudioChannels, + livekit::TrackSource::SOURCE_MICROPHONE); + ASSERT_NE(audio_track, nullptr); + + const std::string caller_identity = "rpc-caller"; + + std::atomic frames_sent{0}; + std::atomic frames_received{0}; + std::atomic push_failures{0}; + std::atomic running{true}; + + receiver.setOnAudioFrameCallback( + caller_identity, livekit::TrackSource::SOURCE_MICROPHONE, + [&](const livekit::AudioFrame &) { frames_received++; }); + + std::this_thread::sleep_for(3s); + + auto start_time = std::chrono::steady_clock::now(); + auto duration = std::chrono::seconds(config_.stress_duration_seconds); + + std::thread sender([&]() { + int phase = 0; + auto next_frame_time = std::chrono::steady_clock::now(); + const auto frame_duration = + std::chrono::milliseconds(kStressFrameDurationMs); + + while (running.load()) { + std::this_thread::sleep_until(next_frame_time); + next_frame_time += frame_duration; + + auto data = makeSineFrame(kStressSamplesPerFrame, 440.0, phase); + bool ok = audio_track->pushFrame(data, kStressSamplesPerFrame); + if (ok) { + frames_sent++; + } else { + push_failures++; + } + } + }); + + std::thread progress([&]() { + int64_t last_sent = 0; + int64_t last_received = 0; + while (running.load()) { + std::this_thread::sleep_for(30s); + if (!running.load()) + break; + + auto elapsed = std::chrono::steady_clock::now() - start_time; + auto elapsed_s = + std::chrono::duration_cast(elapsed).count(); + int64_t cur_sent = frames_sent.load(); + int64_t cur_received = frames_received.load(); + double send_rate = (cur_sent - last_sent) / 30.0; + double recv_rate = (cur_received - last_received) / 30.0; + last_sent = cur_sent; + last_received = cur_received; + + std::cout << "[" << elapsed_s << "s]" + << " sent=" << cur_sent << " recv=" << cur_received + << " failures=" << push_failures.load() + << " send_rate=" << std::fixed << std::setprecision(1) + << send_rate << "/s" + << " recv_rate=" << recv_rate << "/s" << std::endl; + } + }); + + while (std::chrono::steady_clock::now() - start_time < duration) { + std::this_thread::sleep_for(1s); + } + + running.store(false); + sender.join(); + progress.join(); + + std::this_thread::sleep_for(2s); + + std::cout << "\n========================================" << std::endl; + std::cout << " Sustained Audio Stress Results" << std::endl; + std::cout << "========================================" << std::endl; + std::cout << "Duration: " << config_.stress_duration_seconds << "s" + << std::endl; + std::cout << "Frames sent: " << frames_sent.load() << std::endl; + std::cout << "Frames received: " << frames_received.load() << std::endl; + std::cout << "Push failures: " << push_failures.load() << std::endl; + + double delivery_rate = + frames_sent.load() > 0 + ? (100.0 * frames_received.load() / frames_sent.load()) + : 0.0; + std::cout << "Delivery rate: " << std::fixed << std::setprecision(2) + << delivery_rate << "%" << std::endl; + std::cout << "========================================\n" << std::endl; + + EXPECT_GT(frames_sent.load(), 0) << "Should have sent frames"; + EXPECT_GT(frames_received.load(), 0) << "Should have received frames"; + EXPECT_EQ(push_failures.load(), 0) << "No push failures expected"; + EXPECT_GT(delivery_rate, 50.0) << "Delivery rate below 50%"; + + receiver.clearOnAudioFrameCallback( + caller_identity, livekit::TrackSource::SOURCE_MICROPHONE); +} + +// --------------------------------------------------------------------------- +// Track release under active push: one thread pushes audio continuously +// while another releases the track after a delay. Verifies clean shutdown +// with no crashes or hangs. +// --------------------------------------------------------------------------- +TEST_F(BridgeAudioStressTest, ReleaseUnderActivePush) { + skipIfNotConfigured(); + + std::cout << "\n=== Bridge Audio Release Under Active Push ===" << std::endl; + + const int iterations = config_.test_iterations; + std::cout << "Iterations: " << iterations << std::endl; + + for (int iter = 0; iter < iterations; ++iter) { + { + LiveKitBridge bridge; + livekit::RoomOptions options; + options.auto_subscribe = true; + + bool connected = + bridge.connect(config_.url, config_.caller_token, options); + ASSERT_TRUE(connected) << "Iteration " << iter << ": connect failed"; + + auto track = bridge.createAudioTrack( + "release-stress-mic", kStressAudioSampleRate, kStressAudioChannels, + livekit::TrackSource::SOURCE_MICROPHONE); + + std::atomic pushing{true}; + std::atomic push_count{0}; + + std::thread pusher([&]() { + int phase = 0; + while (pushing.load()) { + auto data = makeSineFrame(kStressSamplesPerFrame, 440.0, phase); + track->pushFrame(data, kStressSamplesPerFrame); + push_count++; + std::this_thread::sleep_for( + std::chrono::milliseconds(kStressFrameDurationMs)); + } + }); + + std::this_thread::sleep_for(500ms); + track->release(); + EXPECT_TRUE(track->isReleased()); + + std::this_thread::sleep_for(200ms); + pushing.store(false); + pusher.join(); + + std::cout << " Iteration " << (iter + 1) << "/" << iterations + << " OK (pushed " << push_count.load() << " frames)" + << std::endl; + } // bridge destroyed here + + std::this_thread::sleep_for(1s); + } +} + +// --------------------------------------------------------------------------- +// Rapid connect/disconnect with active audio callback. Verifies that the +// bridge's reader thread cleanup handles abrupt disconnection. +// --------------------------------------------------------------------------- +TEST_F(BridgeAudioStressTest, RapidConnectDisconnectWithCallback) { + skipIfNotConfigured(); + + const int cycles = config_.test_iterations; + std::cout << "\n=== Bridge Rapid Connect/Disconnect With Audio Callback ===" + << std::endl; + std::cout << "Cycles: " << cycles << std::endl; + + const std::string caller_identity = "rpc-caller"; + + for (int i = 0; i < cycles; ++i) { + { + LiveKitBridge caller; + LiveKitBridge receiver; + + ASSERT_TRUE(connectPair(caller, receiver)) + << "Cycle " << i << ": connect failed"; + + auto track = caller.createAudioTrack( + "rapid-mic", kStressAudioSampleRate, kStressAudioChannels, + livekit::TrackSource::SOURCE_MICROPHONE); + + std::atomic rx_count{0}; + receiver.setOnAudioFrameCallback( + caller_identity, livekit::TrackSource::SOURCE_MICROPHONE, + [&](const livekit::AudioFrame &) { rx_count++; }); + + int phase = 0; + for (int f = 0; f < 50; ++f) { + auto data = makeSineFrame(kStressSamplesPerFrame, 440.0, phase); + track->pushFrame(data, kStressSamplesPerFrame); + std::this_thread::sleep_for( + std::chrono::milliseconds(kStressFrameDurationMs)); + } + + // Clear callback to join reader thread while rx_count is still alive + receiver.clearOnAudioFrameCallback( + caller_identity, livekit::TrackSource::SOURCE_MICROPHONE); + + std::cout << " Cycle " << (i + 1) << "/" << cycles + << " OK (rx=" << rx_count.load() << ")" << std::endl; + } // both bridges destroyed here + + std::this_thread::sleep_for(1s); + } +} + +} // namespace test +} // namespace livekit_bridge diff --git a/bridge/tests/stress/test_bridge_callback_stress.cpp b/bridge/tests/stress/test_bridge_callback_stress.cpp new file mode 100644 index 0000000..7574a6d --- /dev/null +++ b/bridge/tests/stress/test_bridge_callback_stress.cpp @@ -0,0 +1,277 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../common/bridge_test_common.h" +#include +#include + +namespace livekit_bridge { +namespace test { + +constexpr int kCbSampleRate = 48000; +constexpr int kCbChannels = 1; +constexpr int kCbFrameDurationMs = 10; +constexpr int kCbSamplesPerFrame = + kCbSampleRate * kCbFrameDurationMs / 1000; + +static std::vector cbSilentFrame() { + return std::vector(kCbSamplesPerFrame * kCbChannels, 0); +} + +static std::vector cbPayload(size_t size) { + static thread_local std::mt19937 gen(std::random_device{}()); + std::uniform_int_distribution dist(0, 255); + std::vector buf(size); + for (auto &b : buf) + b = static_cast(dist(gen)); + return buf; +} + +class BridgeCallbackStressTest : public BridgeTestBase {}; + +// --------------------------------------------------------------------------- +// Audio callback churn. +// +// Caller pushes audio at real-time pace. A separate thread rapidly +// registers / clears the receiver's audio callback. Each clear() must +// join the reader thread (which closes the AudioStream), and the +// subsequent register must start a new reader. This hammers the +// extract-thread-outside-lock pattern in clearOnAudioFrameCallback(). +// --------------------------------------------------------------------------- +TEST_F(BridgeCallbackStressTest, AudioCallbackChurn) { + skipIfNotConfigured(); + + std::cout << "\n=== Bridge Audio Callback Churn ===" << std::endl; + + LiveKitBridge caller; + LiveKitBridge receiver; + + ASSERT_TRUE(connectPair(caller, receiver)); + + auto audio = caller.createAudioTrack( + "churn-mic", kCbSampleRate, kCbChannels, + livekit::TrackSource::SOURCE_MICROPHONE); + ASSERT_NE(audio, nullptr); + + const std::string caller_identity = "rpc-caller"; + + std::atomic running{true}; + std::atomic total_rx{0}; + std::atomic churn_cycles{0}; + + std::thread pusher([&]() { + auto next = std::chrono::steady_clock::now(); + while (running.load()) { + std::this_thread::sleep_until(next); + next += std::chrono::milliseconds(kCbFrameDurationMs); + auto frame = cbSilentFrame(); + audio->pushFrame(frame, kCbSamplesPerFrame); + } + }); + + std::thread churner([&]() { + while (running.load()) { + receiver.setOnAudioFrameCallback( + caller_identity, livekit::TrackSource::SOURCE_MICROPHONE, + [&](const livekit::AudioFrame &) { total_rx++; }); + + std::this_thread::sleep_for(300ms); + + receiver.clearOnAudioFrameCallback( + caller_identity, livekit::TrackSource::SOURCE_MICROPHONE); + + std::this_thread::sleep_for(100ms); + churn_cycles++; + } + }); + + const int duration_s = std::min(config_.stress_duration_seconds, 30); + std::this_thread::sleep_for(std::chrono::seconds(duration_s)); + + running.store(false); + pusher.join(); + churner.join(); + + std::cout << "Churn cycles: " << churn_cycles.load() << std::endl; + std::cout << "Audio frames received: " << total_rx.load() << std::endl; + + EXPECT_GT(churn_cycles.load(), 0); +} + +// --------------------------------------------------------------------------- +// Mixed audio + data callback churn. +// +// Caller publishes both an audio and data track. Two independent churn +// threads each toggle their respective callback. A third thread pushes +// frames on both tracks. This exercises the bridge's two independent +// callback maps and reader sets under concurrent mutation. +// --------------------------------------------------------------------------- +TEST_F(BridgeCallbackStressTest, MixedCallbackChurn) { + skipIfNotConfigured(); + + std::cout << "\n=== Bridge Mixed Callback Churn ===" << std::endl; + + LiveKitBridge caller; + LiveKitBridge receiver; + + ASSERT_TRUE(connectPair(caller, receiver)); + + auto audio = caller.createAudioTrack( + "mixed-mic", kCbSampleRate, kCbChannels, + livekit::TrackSource::SOURCE_MICROPHONE); + auto data = caller.createDataTrack("mixed-data"); + ASSERT_NE(audio, nullptr); + ASSERT_NE(data, nullptr); + + const std::string caller_identity = "rpc-caller"; + + std::atomic running{true}; + std::atomic audio_rx{0}; + std::atomic data_rx{0}; + std::atomic audio_churns{0}; + std::atomic data_churns{0}; + + std::thread audio_pusher([&]() { + auto next = std::chrono::steady_clock::now(); + while (running.load()) { + std::this_thread::sleep_until(next); + next += std::chrono::milliseconds(kCbFrameDurationMs); + auto frame = cbSilentFrame(); + audio->pushFrame(frame, kCbSamplesPerFrame); + } + }); + + std::thread data_pusher([&]() { + while (running.load()) { + auto payload = cbPayload(256); + data->pushFrame(payload); + std::this_thread::sleep_for(20ms); + } + }); + + std::thread audio_churner([&]() { + while (running.load()) { + receiver.setOnAudioFrameCallback( + caller_identity, livekit::TrackSource::SOURCE_MICROPHONE, + [&](const livekit::AudioFrame &) { audio_rx++; }); + std::this_thread::sleep_for(250ms); + receiver.clearOnAudioFrameCallback( + caller_identity, livekit::TrackSource::SOURCE_MICROPHONE); + std::this_thread::sleep_for(100ms); + audio_churns++; + } + }); + + std::thread data_churner([&]() { + while (running.load()) { + receiver.setOnDataFrameCallback( + caller_identity, "mixed-data", + [&](const std::vector &, + std::optional) { data_rx++; }); + std::this_thread::sleep_for(350ms); + receiver.clearOnDataFrameCallback(caller_identity, "mixed-data"); + std::this_thread::sleep_for(150ms); + data_churns++; + } + }); + + const int duration_s = std::min(config_.stress_duration_seconds, 30); + std::this_thread::sleep_for(std::chrono::seconds(duration_s)); + + running.store(false); + audio_pusher.join(); + data_pusher.join(); + audio_churner.join(); + data_churner.join(); + + std::cout << "Audio churn cycles: " << audio_churns.load() << std::endl; + std::cout << "Data churn cycles: " << data_churns.load() << std::endl; + std::cout << "Audio frames rx: " << audio_rx.load() << std::endl; + std::cout << "Data frames rx: " << data_rx.load() << std::endl; + + EXPECT_GT(audio_churns.load(), 0); + EXPECT_GT(data_churns.load(), 0); +} + +// --------------------------------------------------------------------------- +// Callback replacement storm. +// +// Instead of clear + set, rapidly replace the callback with a new lambda +// (calling setOnAudioFrameCallback twice without a clear in between). +// The bridge should silently overwrite the old callback and the new one +// should start receiving. The old reader thread should eventually be +// replaced the next time onTrackSubscribed fires. +// --------------------------------------------------------------------------- +TEST_F(BridgeCallbackStressTest, CallbackReplacement) { + skipIfNotConfigured(); + + std::cout << "\n=== Bridge Callback Replacement Storm ===" << std::endl; + + LiveKitBridge caller; + LiveKitBridge receiver; + + ASSERT_TRUE(connectPair(caller, receiver)); + + auto audio = caller.createAudioTrack( + "replace-mic", kCbSampleRate, kCbChannels, + livekit::TrackSource::SOURCE_MICROPHONE); + ASSERT_NE(audio, nullptr); + + const std::string caller_identity = "rpc-caller"; + + std::atomic running{true}; + std::atomic total_rx{0}; + std::atomic replacements{0}; + + std::thread pusher([&]() { + auto next = std::chrono::steady_clock::now(); + while (running.load()) { + std::this_thread::sleep_until(next); + next += std::chrono::milliseconds(kCbFrameDurationMs); + auto frame = cbSilentFrame(); + audio->pushFrame(frame, kCbSamplesPerFrame); + } + }); + + std::thread replacer([&]() { + while (running.load()) { + receiver.setOnAudioFrameCallback( + caller_identity, livekit::TrackSource::SOURCE_MICROPHONE, + [&](const livekit::AudioFrame &) { total_rx++; }); + replacements++; + std::this_thread::sleep_for(100ms); + } + }); + + const int duration_s = std::min(config_.stress_duration_seconds, 20); + std::this_thread::sleep_for(std::chrono::seconds(duration_s)); + + running.store(false); + pusher.join(); + replacer.join(); + + // Final clear to join any lingering reader + receiver.clearOnAudioFrameCallback( + caller_identity, livekit::TrackSource::SOURCE_MICROPHONE); + + std::cout << "Replacements: " << replacements.load() << std::endl; + std::cout << "Total frames rx: " << total_rx.load() << std::endl; + + EXPECT_GT(replacements.load(), 0); +} + +} // namespace test +} // namespace livekit_bridge diff --git a/bridge/tests/stress/test_bridge_data_stress.cpp b/bridge/tests/stress/test_bridge_data_stress.cpp new file mode 100644 index 0000000..23937df --- /dev/null +++ b/bridge/tests/stress/test_bridge_data_stress.cpp @@ -0,0 +1,337 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../common/bridge_test_common.h" +#include + +namespace livekit_bridge { +namespace test { + +static std::vector randomPayload(size_t size) { + static thread_local std::mt19937 gen(std::random_device{}()); + std::uniform_int_distribution dist(0, 255); + std::vector data(size); + for (auto &b : data) + b = static_cast(dist(gen)); + return data; +} + +class BridgeDataStressTest : public BridgeTestBase {}; + +// --------------------------------------------------------------------------- +// High-throughput data track stress test. +// +// Pushes data frames as fast as possible for STRESS_DURATION_SECONDS and +// tracks throughput, delivery rate, and back-pressure failures. +// --------------------------------------------------------------------------- +TEST_F(BridgeDataStressTest, HighThroughput) { + skipIfNotConfigured(); + + constexpr size_t kPayloadSize = 1024; + + std::cout << "\n=== Bridge Data High-Throughput Stress Test ===" << std::endl; + std::cout << "Duration: " << config_.stress_duration_seconds << "s" + << std::endl; + std::cout << "Payload size: " << kPayloadSize << " bytes" << std::endl; + + LiveKitBridge caller; + LiveKitBridge receiver; + + ASSERT_TRUE(connectPair(caller, receiver)); + + const std::string track_name = "throughput-data"; + const std::string caller_identity = "rpc-caller"; + + auto data_track = caller.createDataTrack(track_name); + ASSERT_NE(data_track, nullptr); + + StressTestStats stats; + std::atomic frames_received{0}; + std::atomic running{true}; + + receiver.setOnDataFrameCallback( + caller_identity, track_name, + [&](const std::vector &payload, + std::optional) { + frames_received++; + (void)payload; + }); + + std::this_thread::sleep_for(3s); + + auto start_time = std::chrono::steady_clock::now(); + auto duration = std::chrono::seconds(config_.stress_duration_seconds); + + std::thread sender([&]() { + while (running.load()) { + auto payload = randomPayload(kPayloadSize); + + auto t0 = std::chrono::high_resolution_clock::now(); + bool ok = data_track->pushFrame(payload); + auto t1 = std::chrono::high_resolution_clock::now(); + + double latency_ms = + std::chrono::duration(t1 - t0).count(); + + if (ok) { + stats.recordCall(true, latency_ms, kPayloadSize); + } else { + stats.recordCall(false, latency_ms, kPayloadSize); + stats.recordError("push_failed"); + } + + std::this_thread::sleep_for(5ms); + } + }); + + std::thread progress([&]() { + int last_total = 0; + while (running.load()) { + std::this_thread::sleep_for(30s); + if (!running.load()) + break; + + auto elapsed = std::chrono::steady_clock::now() - start_time; + auto elapsed_s = + std::chrono::duration_cast(elapsed).count(); + int cur_total = stats.totalCalls(); + int rate = (cur_total - last_total); + last_total = cur_total; + + std::cout << "[" << elapsed_s << "s]" + << " sent=" << cur_total + << " recv=" << frames_received.load() + << " success=" << stats.successfulCalls() + << " failed=" << stats.failedCalls() + << " rate=" << std::fixed << std::setprecision(1) + << (rate / 30.0) << " pushes/s" << std::endl; + } + }); + + while (std::chrono::steady_clock::now() - start_time < duration) { + std::this_thread::sleep_for(1s); + } + + running.store(false); + sender.join(); + progress.join(); + + std::this_thread::sleep_for(2s); + + stats.printStats("Bridge Data High-Throughput Stress"); + + std::cout << "Frames received: " << frames_received.load() << std::endl; + + EXPECT_GT(stats.successfulCalls(), 0) << "No successful pushes"; + double success_rate = + stats.totalCalls() > 0 + ? (100.0 * stats.successfulCalls() / stats.totalCalls()) + : 0.0; + EXPECT_GT(success_rate, 95.0) << "Push success rate below 95%"; + + double delivery_rate = + stats.successfulCalls() > 0 + ? (100.0 * frames_received.load() / stats.successfulCalls()) + : 0.0; + std::cout << "Delivery rate: " << std::fixed << std::setprecision(2) + << delivery_rate << "%" << std::endl; + + receiver.clearOnDataFrameCallback(caller_identity, track_name); +} + +// --------------------------------------------------------------------------- +// Large payload stress: pushes 64KB payloads for the configured duration. +// Exercises serialization / deserialization with larger frames. +// --------------------------------------------------------------------------- +TEST_F(BridgeDataStressTest, LargePayloadStress) { + skipIfNotConfigured(); + + constexpr size_t kLargePayloadSize = 64 * 1024; + + std::cout << "\n=== Bridge Data Large-Payload Stress Test ===" << std::endl; + std::cout << "Duration: " << config_.stress_duration_seconds << "s" + << std::endl; + std::cout << "Payload size: " << kLargePayloadSize << " bytes (64KB)" + << std::endl; + + LiveKitBridge caller; + LiveKitBridge receiver; + + ASSERT_TRUE(connectPair(caller, receiver)); + + const std::string track_name = "large-data"; + const std::string caller_identity = "rpc-caller"; + + auto data_track = caller.createDataTrack(track_name); + ASSERT_NE(data_track, nullptr); + + StressTestStats stats; + std::atomic frames_received{0}; + std::atomic bytes_received{0}; + std::atomic running{true}; + + receiver.setOnDataFrameCallback( + caller_identity, track_name, + [&](const std::vector &payload, + std::optional) { + frames_received++; + bytes_received += payload.size(); + }); + + std::this_thread::sleep_for(3s); + + auto start_time = std::chrono::steady_clock::now(); + auto duration = std::chrono::seconds(config_.stress_duration_seconds); + + std::thread sender([&]() { + while (running.load()) { + auto payload = randomPayload(kLargePayloadSize); + + auto t0 = std::chrono::high_resolution_clock::now(); + bool ok = data_track->pushFrame(payload); + auto t1 = std::chrono::high_resolution_clock::now(); + + double latency_ms = + std::chrono::duration(t1 - t0).count(); + + if (ok) { + stats.recordCall(true, latency_ms, kLargePayloadSize); + } else { + stats.recordCall(false, latency_ms, kLargePayloadSize); + stats.recordError("push_failed"); + } + + std::this_thread::sleep_for(50ms); + } + }); + + std::thread progress([&]() { + int last_total = 0; + while (running.load()) { + std::this_thread::sleep_for(30s); + if (!running.load()) + break; + + auto elapsed = std::chrono::steady_clock::now() - start_time; + auto elapsed_s = + std::chrono::duration_cast(elapsed).count(); + int cur_total = stats.totalCalls(); + + std::cout << "[" << elapsed_s << "s]" + << " sent=" << cur_total + << " recv=" << frames_received.load() + << " bytes_rx=" << (bytes_received.load() / (1024.0 * 1024.0)) + << " MB" + << " rate=" << std::fixed << std::setprecision(1) + << ((cur_total - last_total) / 30.0) << " pushes/s" + << std::endl; + last_total = cur_total; + } + }); + + while (std::chrono::steady_clock::now() - start_time < duration) { + std::this_thread::sleep_for(1s); + } + + running.store(false); + sender.join(); + progress.join(); + + std::this_thread::sleep_for(2s); + + stats.printStats("Bridge Data Large-Payload Stress"); + + std::cout << "Frames received: " << frames_received.load() << std::endl; + std::cout << "Bytes received: " + << (bytes_received.load() / (1024.0 * 1024.0)) << " MB" + << std::endl; + + EXPECT_GT(stats.successfulCalls(), 0) << "No successful pushes"; + double success_rate = + stats.totalCalls() > 0 + ? (100.0 * stats.successfulCalls() / stats.totalCalls()) + : 0.0; + EXPECT_GT(success_rate, 90.0) << "Push success rate below 90%"; + + receiver.clearOnDataFrameCallback(caller_identity, track_name); +} + +// --------------------------------------------------------------------------- +// Callback churn: rapidly register/unregister the data frame callback while +// the sender is actively pushing. Exercises the bridge's thread-joining +// logic under contention. +// --------------------------------------------------------------------------- +TEST_F(BridgeDataStressTest, CallbackChurn) { + skipIfNotConfigured(); + + std::cout << "\n=== Bridge Data Callback Churn Stress Test ===" << std::endl; + + LiveKitBridge caller; + LiveKitBridge receiver; + + ASSERT_TRUE(connectPair(caller, receiver)); + + const std::string track_name = "churn-data"; + const std::string caller_identity = "rpc-caller"; + + auto data_track = caller.createDataTrack(track_name); + ASSERT_NE(data_track, nullptr); + + std::atomic running{true}; + std::atomic total_received{0}; + std::atomic churn_cycles{0}; + + std::thread sender([&]() { + while (running.load()) { + auto payload = randomPayload(256); + data_track->pushFrame(payload); + std::this_thread::sleep_for(10ms); + } + }); + + std::thread churner([&]() { + while (running.load()) { + receiver.setOnDataFrameCallback( + caller_identity, track_name, + [&](const std::vector &, + std::optional) { total_received++; }); + + std::this_thread::sleep_for(500ms); + + receiver.clearOnDataFrameCallback(caller_identity, track_name); + + std::this_thread::sleep_for(200ms); + churn_cycles++; + } + }); + + const int churn_duration_s = std::min(config_.stress_duration_seconds, 30); + std::this_thread::sleep_for(std::chrono::seconds(churn_duration_s)); + + running.store(false); + sender.join(); + churner.join(); + + std::cout << "Churn cycles completed: " << churn_cycles.load() << std::endl; + std::cout << "Total frames received: " << total_received.load() + << std::endl; + + EXPECT_GT(churn_cycles.load(), 0) + << "Should have completed at least one churn cycle"; +} + +} // namespace test +} // namespace livekit_bridge diff --git a/bridge/tests/stress/test_bridge_lifecycle_stress.cpp b/bridge/tests/stress/test_bridge_lifecycle_stress.cpp new file mode 100644 index 0000000..4d37161 --- /dev/null +++ b/bridge/tests/stress/test_bridge_lifecycle_stress.cpp @@ -0,0 +1,298 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../common/bridge_test_common.h" +#include +#include + +namespace livekit_bridge { +namespace test { + +constexpr int kLifecycleSampleRate = 48000; +constexpr int kLifecycleChannels = 1; +constexpr int kLifecycleFrameDurationMs = 10; +constexpr int kLifecycleSamplesPerFrame = + kLifecycleSampleRate * kLifecycleFrameDurationMs / 1000; + +static std::vector makeSilent(int samples) { + return std::vector(samples * kLifecycleChannels, 0); +} + +static std::vector makePayload(size_t size) { + static thread_local std::mt19937 gen(std::random_device{}()); + std::uniform_int_distribution dist(0, 255); + std::vector buf(size); + for (auto &b : buf) + b = static_cast(dist(gen)); + return buf; +} + +class BridgeLifecycleStressTest : public BridgeTestBase {}; + +// --------------------------------------------------------------------------- +// Disconnect while frames are actively being pushed and received. +// +// Two threads push audio / data frames at full rate. A third thread +// triggers disconnect() mid-flight after a random delay. The bridge must +// tear down cleanly with no crashes, hangs, or thread leaks. Repeated +// for TEST_ITERATIONS cycles. +// --------------------------------------------------------------------------- +TEST_F(BridgeLifecycleStressTest, DisconnectUnderLoad) { + skipIfNotConfigured(); + + const int cycles = config_.test_iterations; + std::cout << "\n=== Bridge Disconnect Under Load ===" << std::endl; + std::cout << "Cycles: " << cycles << std::endl; + + const std::string caller_identity = "rpc-caller"; + + for (int i = 0; i < cycles; ++i) { + { + LiveKitBridge caller; + LiveKitBridge receiver; + + ASSERT_TRUE(connectPair(caller, receiver)) + << "Cycle " << i << ": connect failed"; + + auto audio = caller.createAudioTrack( + "load-mic", kLifecycleSampleRate, kLifecycleChannels, + livekit::TrackSource::SOURCE_MICROPHONE); + auto data = caller.createDataTrack("load-data"); + + std::atomic audio_rx{0}; + std::atomic data_rx{0}; + + receiver.setOnAudioFrameCallback( + caller_identity, livekit::TrackSource::SOURCE_MICROPHONE, + [&](const livekit::AudioFrame &) { audio_rx++; }); + + receiver.setOnDataFrameCallback( + caller_identity, "load-data", + [&](const std::vector &, + std::optional) { data_rx++; }); + + std::this_thread::sleep_for(2s); + + std::atomic running{true}; + + std::thread audio_pusher([&]() { + while (running.load()) { + auto frame = makeSilent(kLifecycleSamplesPerFrame); + audio->pushFrame(frame, kLifecycleSamplesPerFrame); + std::this_thread::sleep_for( + std::chrono::milliseconds(kLifecycleFrameDurationMs)); + } + }); + + std::thread data_pusher([&]() { + while (running.load()) { + auto payload = makePayload(512); + data->pushFrame(payload); + std::this_thread::sleep_for(20ms); + } + }); + + // Let traffic flow for 1-2 seconds, then pull the plug. + std::this_thread::sleep_for(1500ms); + running.store(false); + audio_pusher.join(); + data_pusher.join(); + + // Clear callbacks while captured atomics are still alive + receiver.clearOnAudioFrameCallback( + caller_identity, livekit::TrackSource::SOURCE_MICROPHONE); + receiver.clearOnDataFrameCallback(caller_identity, "load-data"); + + std::cout << " Cycle " << (i + 1) << "/" << cycles + << " OK (audio_rx=" << audio_rx.load() + << " data_rx=" << data_rx.load() << ")" << std::endl; + } // bridges destroyed → disconnect + shutdown + + std::this_thread::sleep_for(1s); + } +} + +// --------------------------------------------------------------------------- +// Track release while receiver is consuming. +// +// Caller publishes audio + data, receiver is actively consuming via +// callbacks, then caller releases each track individually while the +// receiver's reader threads are still running. Verifies no dangling +// pointers, no use-after-free, and clean thread join. +// --------------------------------------------------------------------------- +TEST_F(BridgeLifecycleStressTest, TrackReleaseWhileReceiving) { + skipIfNotConfigured(); + + const int iterations = config_.test_iterations; + std::cout << "\n=== Bridge Track Release While Receiving ===" << std::endl; + std::cout << "Iterations: " << iterations << std::endl; + + const std::string caller_identity = "rpc-caller"; + + for (int iter = 0; iter < iterations; ++iter) { + { + LiveKitBridge caller; + LiveKitBridge receiver; + + ASSERT_TRUE(connectPair(caller, receiver)) + << "Iteration " << iter << ": connect failed"; + + auto audio = caller.createAudioTrack( + "release-rx-mic", kLifecycleSampleRate, kLifecycleChannels, + livekit::TrackSource::SOURCE_MICROPHONE); + auto data = caller.createDataTrack("release-rx-data"); + + std::atomic audio_rx{0}; + std::atomic data_rx{0}; + + receiver.setOnAudioFrameCallback( + caller_identity, livekit::TrackSource::SOURCE_MICROPHONE, + [&](const livekit::AudioFrame &) { audio_rx++; }); + + receiver.setOnDataFrameCallback( + caller_identity, "release-rx-data", + [&](const std::vector &, + std::optional) { data_rx++; }); + + std::this_thread::sleep_for(2s); + + std::atomic running{true}; + + std::thread audio_pusher([&]() { + while (running.load()) { + if (audio->isReleased()) + break; + auto frame = makeSilent(kLifecycleSamplesPerFrame); + audio->pushFrame(frame, kLifecycleSamplesPerFrame); + std::this_thread::sleep_for( + std::chrono::milliseconds(kLifecycleFrameDurationMs)); + } + }); + + std::thread data_pusher([&]() { + while (running.load()) { + if (data->isReleased()) + break; + auto payload = makePayload(256); + data->pushFrame(payload); + std::this_thread::sleep_for(20ms); + } + }); + + // Let frames flow, then release tracks mid-stream + std::this_thread::sleep_for(800ms); + + audio->release(); + EXPECT_TRUE(audio->isReleased()); + + std::this_thread::sleep_for(200ms); + + data->release(); + EXPECT_TRUE(data->isReleased()); + + running.store(false); + audio_pusher.join(); + data_pusher.join(); + + // pushFrame must return false on released tracks + auto silence = makeSilent(kLifecycleSamplesPerFrame); + EXPECT_FALSE(audio->pushFrame(silence, kLifecycleSamplesPerFrame)); + + auto payload = makePayload(64); + EXPECT_FALSE(data->pushFrame(payload)); + + receiver.clearOnAudioFrameCallback( + caller_identity, livekit::TrackSource::SOURCE_MICROPHONE); + receiver.clearOnDataFrameCallback(caller_identity, "release-rx-data"); + + std::cout << " Iteration " << (iter + 1) << "/" << iterations + << " OK (audio_rx=" << audio_rx.load() + << " data_rx=" << data_rx.load() << ")" << std::endl; + } + + std::this_thread::sleep_for(1s); + } +} + +// --------------------------------------------------------------------------- +// Repeated full lifecycle: connect → create all track types → push → +// release → disconnect. Exercises the complete resource creation and +// teardown path looking for accumulating leaks. +// --------------------------------------------------------------------------- +TEST_F(BridgeLifecycleStressTest, FullLifecycleSoak) { + skipIfNotConfigured(); + + const int cycles = config_.test_iterations; + std::cout << "\n=== Bridge Full Lifecycle Soak ===" << std::endl; + std::cout << "Cycles: " << cycles << std::endl; + + for (int i = 0; i < cycles; ++i) { + { + LiveKitBridge bridge; + livekit::RoomOptions options; + options.auto_subscribe = true; + + bool connected = + bridge.connect(config_.url, config_.caller_token, options); + ASSERT_TRUE(connected) << "Cycle " << i << ": connect failed"; + + auto audio = bridge.createAudioTrack( + "soak-mic", kLifecycleSampleRate, kLifecycleChannels, + livekit::TrackSource::SOURCE_MICROPHONE); + + constexpr int kVideoWidth = 320; + constexpr int kVideoHeight = 240; + auto video = bridge.createVideoTrack( + "soak-cam", kVideoWidth, kVideoHeight, + livekit::TrackSource::SOURCE_CAMERA); + + auto data = bridge.createDataTrack("soak-data"); + + // Push a handful of frames on each track type + for (int f = 0; f < 10; ++f) { + auto pcm = makeSilent(kLifecycleSamplesPerFrame); + audio->pushFrame(pcm, kLifecycleSamplesPerFrame); + + std::vector rgba(kVideoWidth * kVideoHeight * 4, 0x80); + video->pushFrame(rgba); + + auto payload = makePayload(256); + data->pushFrame(payload); + + std::this_thread::sleep_for( + std::chrono::milliseconds(kLifecycleFrameDurationMs)); + } + + // Explicit release in various orders to exercise different teardown paths + if (i % 3 == 0) { + audio->release(); + video->release(); + data->release(); + } else if (i % 3 == 1) { + data->release(); + audio->release(); + video->release(); + } + // else: let disconnect() release all tracks + } // bridge destroyed + + std::cout << " Cycle " << (i + 1) << "/" << cycles << " OK" << std::endl; + std::this_thread::sleep_for(1s); + } +} + +} // namespace test +} // namespace livekit_bridge diff --git a/bridge/tests/stress/test_bridge_multi_track_stress.cpp b/bridge/tests/stress/test_bridge_multi_track_stress.cpp new file mode 100644 index 0000000..c54d2ca --- /dev/null +++ b/bridge/tests/stress/test_bridge_multi_track_stress.cpp @@ -0,0 +1,440 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../common/bridge_test_common.h" +#include +#include + +namespace livekit_bridge { +namespace test { + +constexpr int kMtSampleRate = 48000; +constexpr int kMtChannels = 1; +constexpr int kMtFrameDurationMs = 10; +constexpr int kMtSamplesPerFrame = kMtSampleRate * kMtFrameDurationMs / 1000; +constexpr int kMtVideoWidth = 160; +constexpr int kMtVideoHeight = 120; +constexpr size_t kMtVideoFrameBytes = kMtVideoWidth * kMtVideoHeight * 4; + +static std::vector mtSilentFrame() { + return std::vector(kMtSamplesPerFrame * kMtChannels, 0); +} + +static std::vector mtVideoFrame() { + return std::vector(kMtVideoFrameBytes, 0x42); +} + +static std::vector mtPayload(size_t size) { + static thread_local std::mt19937 gen(std::random_device{}()); + std::uniform_int_distribution dist(0, 255); + std::vector buf(size); + for (auto &b : buf) + b = static_cast(dist(gen)); + return buf; +} + +class BridgeMultiTrackStressTest : public BridgeTestBase {}; + +// --------------------------------------------------------------------------- +// Concurrent pushes on all track types. +// +// Publishes an audio track, a video track, and two data tracks. A +// separate thread pushes frames on each track simultaneously for the +// configured stress duration. All four threads contend on the bridge's +// internal mutex and on the underlying FFI. Reports per-track push +// success rates. +// --------------------------------------------------------------------------- +TEST_F(BridgeMultiTrackStressTest, ConcurrentMultiTrackPush) { + skipIfNotConfigured(); + + std::cout << "\n=== Bridge Concurrent Multi-Track Push ===" << std::endl; + std::cout << "Duration: " << config_.stress_duration_seconds << "s" + << std::endl; + + LiveKitBridge bridge; + livekit::RoomOptions options; + options.auto_subscribe = true; + + bool connected = bridge.connect(config_.url, config_.caller_token, options); + ASSERT_TRUE(connected); + + auto audio = bridge.createAudioTrack( + "mt-mic", kMtSampleRate, kMtChannels, + livekit::TrackSource::SOURCE_MICROPHONE); + + auto video = bridge.createVideoTrack( + "mt-cam", kMtVideoWidth, kMtVideoHeight, + livekit::TrackSource::SOURCE_CAMERA); + + auto data1 = bridge.createDataTrack("mt-data-1"); + auto data2 = bridge.createDataTrack("mt-data-2"); + + ASSERT_NE(audio, nullptr); + ASSERT_NE(video, nullptr); + ASSERT_NE(data1, nullptr); + ASSERT_NE(data2, nullptr); + + struct TrackStats { + std::atomic pushes{0}; + std::atomic successes{0}; + std::atomic failures{0}; + }; + + TrackStats audio_stats, video_stats, data1_stats, data2_stats; + std::atomic running{true}; + + auto start_time = std::chrono::steady_clock::now(); + auto duration = std::chrono::seconds(config_.stress_duration_seconds); + + std::thread audio_thread([&]() { + auto next = std::chrono::steady_clock::now(); + while (running.load()) { + std::this_thread::sleep_until(next); + next += std::chrono::milliseconds(kMtFrameDurationMs); + auto frame = mtSilentFrame(); + bool ok = audio->pushFrame(frame, kMtSamplesPerFrame); + audio_stats.pushes++; + if (ok) + audio_stats.successes++; + else + audio_stats.failures++; + } + }); + + std::thread video_thread([&]() { + while (running.load()) { + auto frame = mtVideoFrame(); + bool ok = video->pushFrame(frame); + video_stats.pushes++; + if (ok) + video_stats.successes++; + else + video_stats.failures++; + // ~30 fps + std::this_thread::sleep_for(33ms); + } + }); + + std::thread data1_thread([&]() { + while (running.load()) { + auto payload = mtPayload(512); + bool ok = data1->pushFrame(payload); + data1_stats.pushes++; + if (ok) + data1_stats.successes++; + else + data1_stats.failures++; + std::this_thread::sleep_for(10ms); + } + }); + + std::thread data2_thread([&]() { + while (running.load()) { + auto payload = mtPayload(2048); + bool ok = data2->pushFrame(payload); + data2_stats.pushes++; + if (ok) + data2_stats.successes++; + else + data2_stats.failures++; + std::this_thread::sleep_for(15ms); + } + }); + + // Progress reporting + std::thread progress([&]() { + while (running.load()) { + std::this_thread::sleep_for(30s); + if (!running.load()) + break; + auto elapsed = std::chrono::steady_clock::now() - start_time; + auto elapsed_s = + std::chrono::duration_cast(elapsed).count(); + std::cout << "[" << elapsed_s << "s]" + << " audio=" << audio_stats.pushes.load() + << " video=" << video_stats.pushes.load() + << " data1=" << data1_stats.pushes.load() + << " data2=" << data2_stats.pushes.load() << std::endl; + } + }); + + while (std::chrono::steady_clock::now() - start_time < duration) { + std::this_thread::sleep_for(1s); + } + + running.store(false); + audio_thread.join(); + video_thread.join(); + data1_thread.join(); + data2_thread.join(); + progress.join(); + + auto printTrack = [](const char *name, const TrackStats &s) { + double rate = s.pushes.load() > 0 + ? (100.0 * s.successes.load() / s.pushes.load()) + : 0.0; + std::cout << " " << name << ": pushes=" << s.pushes.load() + << " ok=" << s.successes.load() + << " fail=" << s.failures.load() << " (" << std::fixed + << std::setprecision(1) << rate << "%)" << std::endl; + }; + + std::cout << "\n========================================" << std::endl; + std::cout << " Multi-Track Push Results" << std::endl; + std::cout << "========================================" << std::endl; + printTrack("audio ", audio_stats); + printTrack("video ", video_stats); + printTrack("data-1", data1_stats); + printTrack("data-2", data2_stats); + std::cout << "========================================\n" << std::endl; + + EXPECT_GT(audio_stats.successes.load(), 0); + EXPECT_GT(video_stats.successes.load(), 0); + EXPECT_GT(data1_stats.successes.load(), 0); + EXPECT_GT(data2_stats.successes.load(), 0); +} + +// --------------------------------------------------------------------------- +// Concurrent track creation and release. +// +// Multiple threads simultaneously create tracks, push a short burst, +// then release them. Exercises the bridge's published_*_tracks_ vectors +// and mutex under heavy concurrent modification. +// --------------------------------------------------------------------------- +TEST_F(BridgeMultiTrackStressTest, ConcurrentCreateRelease) { + skipIfNotConfigured(); + + std::cout << "\n=== Bridge Concurrent Track Create/Release ===" << std::endl; + + LiveKitBridge bridge; + livekit::RoomOptions options; + options.auto_subscribe = true; + + bool connected = bridge.connect(config_.url, config_.caller_token, options); + ASSERT_TRUE(connected); + + std::atomic running{true}; + std::atomic audio_cycles{0}; + std::atomic data_cycles{0}; + std::atomic errors{0}; + + // Each source can only have one active track, so we serialize by source + // but run the two sources concurrently. + + std::thread audio_thread([&]() { + while (running.load()) { + try { + auto track = bridge.createAudioTrack( + "create-release-mic", kMtSampleRate, kMtChannels, + livekit::TrackSource::SOURCE_MICROPHONE); + + for (int i = 0; i < 5; ++i) { + auto frame = mtSilentFrame(); + track->pushFrame(frame, kMtSamplesPerFrame); + std::this_thread::sleep_for( + std::chrono::milliseconds(kMtFrameDurationMs)); + } + + track->release(); + audio_cycles++; + } catch (const std::exception &e) { + errors++; + std::cerr << "Audio create/release error: " << e.what() << std::endl; + } + std::this_thread::sleep_for(200ms); + } + }); + + std::thread data_thread([&]() { + int track_counter = 0; + while (running.load()) { + try { + auto track = bridge.createDataTrack( + "create-release-data-" + std::to_string(track_counter++)); + + for (int i = 0; i < 5; ++i) { + auto payload = mtPayload(128); + track->pushFrame(payload); + std::this_thread::sleep_for(20ms); + } + + track->release(); + data_cycles++; + } catch (const std::exception &e) { + errors++; + std::cerr << "Data create/release error: " << e.what() << std::endl; + } + std::this_thread::sleep_for(200ms); + } + }); + + const int duration_s = std::min(config_.stress_duration_seconds, 30); + std::this_thread::sleep_for(std::chrono::seconds(duration_s)); + + running.store(false); + audio_thread.join(); + data_thread.join(); + + std::cout << "Audio create/release cycles: " << audio_cycles.load() + << std::endl; + std::cout << "Data create/release cycles: " << data_cycles.load() + << std::endl; + std::cout << "Errors: " << errors.load() << std::endl; + + EXPECT_GT(audio_cycles.load(), 0); + EXPECT_GT(data_cycles.load(), 0); + EXPECT_EQ(errors.load(), 0); +} + +// --------------------------------------------------------------------------- +// Full-duplex multi-track. +// +// Both caller and receiver publish audio + data tracks. Both register +// callbacks for the other's tracks. All four push-threads and all four +// reader threads run simultaneously, exercising the bridge's internal +// maps from both the publish side and the subscribe side. +// --------------------------------------------------------------------------- +TEST_F(BridgeMultiTrackStressTest, FullDuplexMultiTrack) { + skipIfNotConfigured(); + + std::cout << "\n=== Bridge Full-Duplex Multi-Track ===" << std::endl; + std::cout << "Duration: " << config_.stress_duration_seconds << "s" + << std::endl; + + LiveKitBridge caller; + LiveKitBridge receiver; + + ASSERT_TRUE(connectPair(caller, receiver)); + + const std::string caller_identity = "rpc-caller"; + const std::string receiver_identity = "rpc-receiver"; + + // Caller publishes + auto caller_audio = caller.createAudioTrack( + "duplex-mic", kMtSampleRate, kMtChannels, + livekit::TrackSource::SOURCE_MICROPHONE); + auto caller_data = caller.createDataTrack("duplex-data-caller"); + + // Receiver publishes + auto receiver_audio = receiver.createAudioTrack( + "duplex-mic", kMtSampleRate, kMtChannels, + livekit::TrackSource::SOURCE_MICROPHONE); + auto receiver_data = receiver.createDataTrack("duplex-data-receiver"); + + // Cross-register callbacks + std::atomic caller_audio_rx{0}; + std::atomic caller_data_rx{0}; + std::atomic receiver_audio_rx{0}; + std::atomic receiver_data_rx{0}; + + caller.setOnAudioFrameCallback( + receiver_identity, livekit::TrackSource::SOURCE_MICROPHONE, + [&](const livekit::AudioFrame &) { caller_audio_rx++; }); + caller.setOnDataFrameCallback( + receiver_identity, "duplex-data-receiver", + [&](const std::vector &, + std::optional) { caller_data_rx++; }); + + receiver.setOnAudioFrameCallback( + caller_identity, livekit::TrackSource::SOURCE_MICROPHONE, + [&](const livekit::AudioFrame &) { receiver_audio_rx++; }); + receiver.setOnDataFrameCallback( + caller_identity, "duplex-data-caller", + [&](const std::vector &, + std::optional) { receiver_data_rx++; }); + + std::this_thread::sleep_for(3s); + + std::atomic running{true}; + auto start_time = std::chrono::steady_clock::now(); + auto duration = std::chrono::seconds(config_.stress_duration_seconds); + + auto audio_push_fn = + [&](std::shared_ptr track) { + auto next = std::chrono::steady_clock::now(); + while (running.load()) { + std::this_thread::sleep_until(next); + next += std::chrono::milliseconds(kMtFrameDurationMs); + auto frame = mtSilentFrame(); + track->pushFrame(frame, kMtSamplesPerFrame); + } + }; + + auto data_push_fn = + [&](std::shared_ptr track) { + while (running.load()) { + auto payload = mtPayload(256); + track->pushFrame(payload); + std::this_thread::sleep_for(20ms); + } + }; + + std::thread t1(audio_push_fn, caller_audio); + std::thread t2(data_push_fn, caller_data); + std::thread t3(audio_push_fn, receiver_audio); + std::thread t4(data_push_fn, receiver_data); + + std::thread progress([&]() { + while (running.load()) { + std::this_thread::sleep_for(30s); + if (!running.load()) + break; + auto elapsed = std::chrono::steady_clock::now() - start_time; + auto elapsed_s = + std::chrono::duration_cast(elapsed).count(); + std::cout << "[" << elapsed_s << "s]" + << " caller_audio_rx=" << caller_audio_rx.load() + << " caller_data_rx=" << caller_data_rx.load() + << " receiver_audio_rx=" << receiver_audio_rx.load() + << " receiver_data_rx=" << receiver_data_rx.load() + << std::endl; + } + }); + + while (std::chrono::steady_clock::now() - start_time < duration) { + std::this_thread::sleep_for(1s); + } + + running.store(false); + t1.join(); + t2.join(); + t3.join(); + t4.join(); + progress.join(); + + std::cout << "\n========================================" << std::endl; + std::cout << " Full-Duplex Multi-Track Results" << std::endl; + std::cout << "========================================" << std::endl; + std::cout << "Caller audio rx: " << caller_audio_rx.load() << std::endl; + std::cout << "Caller data rx: " << caller_data_rx.load() << std::endl; + std::cout << "Receiver audio rx: " << receiver_audio_rx.load() << std::endl; + std::cout << "Receiver data rx: " << receiver_data_rx.load() << std::endl; + std::cout << "========================================\n" << std::endl; + + EXPECT_GT(receiver_audio_rx.load(), 0); + EXPECT_GT(receiver_data_rx.load(), 0); + + // Clear callbacks while atomics are alive + caller.clearOnAudioFrameCallback( + receiver_identity, livekit::TrackSource::SOURCE_MICROPHONE); + caller.clearOnDataFrameCallback(receiver_identity, "duplex-data-receiver"); + receiver.clearOnAudioFrameCallback( + caller_identity, livekit::TrackSource::SOURCE_MICROPHONE); + receiver.clearOnDataFrameCallback(caller_identity, "duplex-data-caller"); +} + +} // namespace test +} // namespace livekit_bridge diff --git a/bridge/tests/test_bridge_audio_track.cpp b/bridge/tests/unit/test_bridge_audio_track.cpp similarity index 99% rename from bridge/tests/test_bridge_audio_track.cpp rename to bridge/tests/unit/test_bridge_audio_track.cpp index 8e7274e..ced5ae0 100644 --- a/bridge/tests/test_bridge_audio_track.cpp +++ b/bridge/tests/unit/test_bridge_audio_track.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2025 LiveKit + * Copyright 2026 LiveKit * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/bridge/tests/unit/test_bridge_data_track.cpp b/bridge/tests/unit/test_bridge_data_track.cpp new file mode 100644 index 0000000..f490cc4 --- /dev/null +++ b/bridge/tests/unit/test_bridge_data_track.cpp @@ -0,0 +1,132 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include +#include +#include + +namespace livekit_bridge { +namespace test { + +class BridgeDataTrackTest : public ::testing::Test { +protected: + /// Create a BridgeDataTrack with a null SDK track for pure-logic testing. + /// The track is usable for accessor and state management tests but + /// pushFrame / isPublished will return false without a real LocalDataTrack. + static BridgeDataTrack createNullTrack(const std::string &name = "data") { + return BridgeDataTrack(name, std::shared_ptr{}, + nullptr); + } +}; + +TEST_F(BridgeDataTrackTest, AccessorsReturnConstructionValues) { + auto track = createNullTrack("sensor-data"); + + EXPECT_EQ(track.name(), "sensor-data") + << "Name should match construction value"; +} + +TEST_F(BridgeDataTrackTest, InitiallyNotReleased) { + auto track = createNullTrack(); + + EXPECT_FALSE(track.isReleased()) + << "Track should not be released immediately after construction"; +} + +TEST_F(BridgeDataTrackTest, ReleaseMarksTrackAsReleased) { + auto track = createNullTrack(); + + track.release(); + + EXPECT_TRUE(track.isReleased()) + << "Track should be released after calling release()"; +} + +TEST_F(BridgeDataTrackTest, DoubleReleaseIsIdempotent) { + auto track = createNullTrack(); + + track.release(); + EXPECT_NO_THROW(track.release()) + << "Calling release() a second time should be a no-op"; + EXPECT_TRUE(track.isReleased()); +} + +TEST_F(BridgeDataTrackTest, PushFrameVectorAfterReleaseReturnsFalse) { + auto track = createNullTrack(); + track.release(); + + std::vector payload = {0x01, 0x02, 0x03}; + + EXPECT_FALSE(track.pushFrame(payload)) + << "pushFrame (vector) on a released track should return false"; +} + +TEST_F(BridgeDataTrackTest, PushFrameVectorWithTimestampAfterReleaseReturnsFalse) { + auto track = createNullTrack(); + track.release(); + + std::vector payload = {0x01, 0x02}; + EXPECT_FALSE(track.pushFrame(payload, 12345u)) + << "pushFrame (vector, timestamp) on a released track should return false"; +} + +TEST_F(BridgeDataTrackTest, PushFrameRawPointerAfterReleaseReturnsFalse) { + auto track = createNullTrack(); + track.release(); + + std::vector payload = {0x01, 0x02, 0x03}; + + EXPECT_FALSE(track.pushFrame(payload.data(), payload.size())) + << "pushFrame (raw pointer) on a released track should return false"; +} + +TEST_F(BridgeDataTrackTest, PushFrameRawPointerWithTimestampAfterReleaseReturnsFalse) { + auto track = createNullTrack(); + track.release(); + + std::uint8_t data[] = {0xab, 0xcd}; + EXPECT_FALSE(track.pushFrame(data, 2, 99999u)) + << "pushFrame (raw pointer, timestamp) on a released track should return false"; +} + +TEST_F(BridgeDataTrackTest, PushFrameWithNullTrackReturnsFalse) { + auto track = createNullTrack(); + + std::vector payload = {0x01}; + EXPECT_FALSE(track.pushFrame(payload)) + << "pushFrame with null underlying track should return false"; +} + +TEST_F(BridgeDataTrackTest, IsPublishedWithNullTrackReturnsFalse) { + auto track = createNullTrack(); + + EXPECT_FALSE(track.isPublished()) + << "isPublished() with null track should return false"; +} + +TEST_F(BridgeDataTrackTest, IsPublishedAfterReleaseReturnsFalse) { + auto track = createNullTrack(); + track.release(); + + EXPECT_FALSE(track.isPublished()) + << "isPublished() after release() should return false"; +} + +} // namespace test +} // namespace livekit_bridge diff --git a/bridge/tests/test_bridge_video_track.cpp b/bridge/tests/unit/test_bridge_video_track.cpp similarity index 99% rename from bridge/tests/test_bridge_video_track.cpp rename to bridge/tests/unit/test_bridge_video_track.cpp index 08517b0..5e64b8d 100644 --- a/bridge/tests/test_bridge_video_track.cpp +++ b/bridge/tests/unit/test_bridge_video_track.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2025 LiveKit + * Copyright 2026 LiveKit * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/bridge/tests/test_callback_key.cpp b/bridge/tests/unit/test_callback_key.cpp similarity index 99% rename from bridge/tests/test_callback_key.cpp rename to bridge/tests/unit/test_callback_key.cpp index 29974f6..d667f7d 100644 --- a/bridge/tests/test_callback_key.cpp +++ b/bridge/tests/unit/test_callback_key.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2025 LiveKit + * Copyright 2026 LiveKit * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/bridge/tests/test_livekit_bridge.cpp b/bridge/tests/unit/test_livekit_bridge.cpp similarity index 99% rename from bridge/tests/test_livekit_bridge.cpp rename to bridge/tests/unit/test_livekit_bridge.cpp index 38b1d7d..0769349 100644 --- a/bridge/tests/test_livekit_bridge.cpp +++ b/bridge/tests/unit/test_livekit_bridge.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2025 LiveKit + * Copyright 2026 LiveKit * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/bridge_human_robot/human.cpp b/examples/bridge_human_robot/human.cpp index 434deca..3010a3c 100644 --- a/examples/bridge_human_robot/human.cpp +++ b/examples/bridge_human_robot/human.cpp @@ -16,15 +16,17 @@ /* * Human example -- receives audio and video frames from a robot in a - * LiveKit room and renders them using SDL3. + * LiveKit room and renders them using SDL3. Also receives data track + * messages ("robot-status") and prints them to stdout. * - * The robot publishes two video tracks and two audio tracks: + * The robot publishes two video tracks, two audio tracks, and a data track: * - "robot-cam" (SOURCE_CAMERA) -- webcam or placeholder * - "robot-sim-frame" (SOURCE_SCREENSHARE) -- simulated diagnostic * frame * - "robot-mic" (SOURCE_MICROPHONE) -- real microphone or * silence * - "robot-sim-audio" (SOURCE_SCREENSHARE_AUDIO) -- simulated siren tone + * - "robot-status" (data track) -- periodic status string * * Press 'w' to play the webcam feed + real mic, or 's' for sim frame + siren. * The selection controls both video and audio simultaneously. @@ -55,10 +57,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -276,6 +280,15 @@ int main(int argc, char *argv[]) { } }); + // ----- set data callback ----- + bridge.setOnDataFrameCallback( + "robot", "robot-status", + [](const std::vector &payload, + std::optional /*user_timestamp*/) { + std::string msg(payload.begin(), payload.end()); + std::cout << "[human] Data from robot: " << msg << "\n"; + }); + // ----- Stdin input thread (for switching when the SDL window is not focused) // ----- std::thread input_thread([&]() { diff --git a/examples/bridge_human_robot/robot.cpp b/examples/bridge_human_robot/robot.cpp index 18d07b9..294b9f4 100644 --- a/examples/bridge_human_robot/robot.cpp +++ b/examples/bridge_human_robot/robot.cpp @@ -16,7 +16,8 @@ /* * Robot example -- streams real webcam video and microphone audio to a - * LiveKit room using SDL3 for hardware capture. + * LiveKit room using SDL3 for hardware capture, and publishes a data + * track ("robot-status") that sends a status string once per second. * * Usage: * robot [--no-mic] @@ -30,7 +31,8 @@ * --join --room my-room --identity robot \ * --valid-for 24h * - * Run alongside the "human" example (which displays the robot's feed). + * Run alongside the "human" example (which displays the robot's feed + * and prints received data messages). */ #include "livekit/audio_frame.h" @@ -390,10 +392,13 @@ int main(int argc, char *argv[]) { auto sim_cam = bridge.createVideoTrack("robot-sim-frame", kSimWidth, kSimHeight, livekit::TrackSource::SOURCE_SCREENSHARE); + + auto data_track = bridge.createDataTrack("robot-status"); + LK_LOG_INFO("[robot] Publishing {} sim audio ({} Hz, {} ch), cam + sim frame " - "({}x{} / {}x{}).", + "({}x{} / {}x{}), data track.", use_mic ? "mic + " : "(no mic) ", kSampleRate, kChannels, kWidth, - kHeight, kSimWidth, kSimHeight); + kHeight, kSimWidth, kSimHeight, data_track->name()); // ----- SDL Mic capture (only when use_mic) ----- // SDLMicSource pulls 10ms frames from the default recording device and @@ -619,6 +624,27 @@ int main(int argc, char *argv[]) { }); LK_LOG_INFO("[robot] Sim audio (siren) track started."); + // ----- Data track: send a status string once per second ----- + std::atomic data_running{true}; + std::thread data_thread([&]() { + std::uint64_t seq = 0; + auto start = std::chrono::steady_clock::now(); + while (data_running.load()) { + auto elapsed_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count(); + std::string msg = "robot status #" + std::to_string(seq) + + " uptime=" + std::to_string(elapsed_ms) + "ms"; + std::vector payload(msg.begin(), msg.end()); + if (!data_track->pushFrame(payload)) { + break; + } + ++seq; + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + }); + std::cout << "[robot] Data track (robot-status) started.\n"; + // ----- Main loop: keep alive + pump SDL events ----- LK_LOG_INFO("[robot] Streaming... press Ctrl-C to stop."); @@ -639,6 +665,7 @@ int main(int argc, char *argv[]) { cam_running.store(false); sim_running.store(false); sim_audio_running.store(false); + data_running.store(false); if (mic_thread.joinable()) mic_thread.join(); if (cam_thread.joinable()) @@ -647,6 +674,8 @@ int main(int argc, char *argv[]) { sim_thread.join(); if (sim_audio_thread.joinable()) sim_audio_thread.join(); + if (data_thread.joinable()) + data_thread.join(); sdl_mic.reset(); sdl_cam.reset(); @@ -654,6 +683,7 @@ int main(int argc, char *argv[]) { sim_audio.reset(); cam.reset(); sim_cam.reset(); + data_track.reset(); bridge.disconnect(); SDL_Quit(); diff --git a/examples/realsense-livekit/realsense-to-mcap/.gitignore b/examples/realsense-livekit/realsense-to-mcap/.gitignore new file mode 100644 index 0000000..68d0afa --- /dev/null +++ b/examples/realsense-livekit/realsense-to-mcap/.gitignore @@ -0,0 +1,3 @@ +build/ +external/ +generated/ diff --git a/examples/realsense-livekit/realsense-to-mcap/BuildFileDescriptorSet.cpp b/examples/realsense-livekit/realsense-to-mcap/BuildFileDescriptorSet.cpp new file mode 100644 index 0000000..33dcfde --- /dev/null +++ b/examples/realsense-livekit/realsense-to-mcap/BuildFileDescriptorSet.cpp @@ -0,0 +1,40 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "BuildFileDescriptorSet.h" + +#include +#include + +google::protobuf::FileDescriptorSet BuildFileDescriptorSet( + const google::protobuf::Descriptor* toplevelDescriptor) { + google::protobuf::FileDescriptorSet fdSet; + std::queue toAdd; + toAdd.push(toplevelDescriptor->file()); + std::unordered_set seenDependencies; + while (!toAdd.empty()) { + const google::protobuf::FileDescriptor* next = toAdd.front(); + toAdd.pop(); + next->CopyTo(fdSet.add_file()); + for (int i = 0; i < next->dependency_count(); ++i) { + const auto* dep = next->dependency(i); + const std::string depName(dep->name()); + if (seenDependencies.find(depName) == seenDependencies.end()) { + seenDependencies.insert(depName); + toAdd.push(dep); + } + } + } + return fdSet; +} diff --git a/examples/realsense-livekit/realsense-to-mcap/BuildFileDescriptorSet.h b/examples/realsense-livekit/realsense-to-mcap/BuildFileDescriptorSet.h new file mode 100644 index 0000000..5d7b7d6 --- /dev/null +++ b/examples/realsense-livekit/realsense-to-mcap/BuildFileDescriptorSet.h @@ -0,0 +1,23 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include + +#include + +google::protobuf::FileDescriptorSet BuildFileDescriptorSet( + const google::protobuf::Descriptor* toplevelDescriptor); diff --git a/examples/realsense-livekit/realsense-to-mcap/CMakeLists.txt b/examples/realsense-livekit/realsense-to-mcap/CMakeLists.txt new file mode 100644 index 0000000..3d20c9c --- /dev/null +++ b/examples/realsense-livekit/realsense-to-mcap/CMakeLists.txt @@ -0,0 +1,143 @@ +# Copyright 2026 LiveKit, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.16) +project(realsense_mcap) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) + +find_package(Protobuf REQUIRED CONFIG) +find_package(realsense2 REQUIRED) +find_package(PkgConfig REQUIRED) +pkg_check_modules(ZSTD REQUIRED IMPORTED_TARGET libzstd) +pkg_check_modules(LZ4 REQUIRED IMPORTED_TARGET liblz4) +find_package(ZLIB REQUIRED) + +set(REALSENSE_MCAP_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +add_executable(realsense_to_mcap + ${REALSENSE_MCAP_DIR}/src/realsense_to_mcap.cpp + ${REALSENSE_MCAP_DIR}/BuildFileDescriptorSet.cpp + ${REALSENSE_MCAP_DIR}/generated/foxglove/RawImage.pb.cc +) + +target_include_directories(realsense_to_mcap PRIVATE + ${realsense2_INCLUDE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/external/mcap/cpp/mcap/include + ${CMAKE_CURRENT_SOURCE_DIR}/generated + ${CMAKE_CURRENT_SOURCE_DIR} +) + +target_link_libraries(realsense_to_mcap + protobuf::libprotobuf + ${realsense2_LIBRARY} + PkgConfig::ZSTD + PkgConfig::LZ4 +) + +# LiveKit participants: auto-detect SDK build when this tree lives under client-sdk-cpp (e.g. examples/realsense-livekit/realsense-to-mcap) +set(LiveKitBuild_DIR "" CACHE PATH "Path to LiveKit SDK build directory (optional; auto-detected when under client-sdk-cpp)") +if(NOT LiveKitBuild_DIR) + get_filename_component(SDK_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../../.." REALPATH) + if(EXISTS "${SDK_ROOT}/build-release/lib") + set(LiveKitBuild_DIR "${SDK_ROOT}/build-release") + elseif(EXISTS "${SDK_ROOT}/build/lib") + set(LiveKitBuild_DIR "${SDK_ROOT}/build") + endif() +endif() + +get_filename_component(LiveKitSource_DIR "${LiveKitBuild_DIR}" DIRECTORY) +set(LIVEKIT_LIB_DIR "${LiveKitBuild_DIR}/lib") +if(NOT LiveKitBuild_DIR OR NOT EXISTS "${LIVEKIT_LIB_DIR}") + message(FATAL_ERROR "LiveKit SDK build not found. Build the client-sdk-cpp repo from its root (e.g. ./build.sh release), or set -DLiveKitBuild_DIR=.") +endif() +find_library(LIVEKIT_BRIDGE_LIB livekit_bridge PATHS "${LIVEKIT_LIB_DIR}" NO_DEFAULT_PATH) +find_library(LIVEKIT_LIB livekit PATHS "${LIVEKIT_LIB_DIR}" NO_DEFAULT_PATH) +if(NOT LIVEKIT_BRIDGE_LIB) + message(FATAL_ERROR "Could not find liblivekit_bridge in ${LIVEKIT_LIB_DIR}. Build the SDK bridge first.") +endif() +if(NOT LIVEKIT_LIB) + message(FATAL_ERROR "Could not find liblivekit in ${LIVEKIT_LIB_DIR}.") +endif() +set(LIVEKIT_BRIDGE_INCLUDE "${LiveKitSource_DIR}/bridge/include") +set(LIVEKIT_INCLUDE "${LiveKitSource_DIR}/include") +if(NOT EXISTS "${LIVEKIT_BRIDGE_INCLUDE}/livekit_bridge/livekit_bridge.h") + message(FATAL_ERROR "LiveKit bridge headers not found at ${LIVEKIT_BRIDGE_INCLUDE}") +endif() + +# realsense_rgbd: captures RealSense, publishes RGB as video + depth/pose as DataTracks +add_executable(realsense_rgbd + ${REALSENSE_MCAP_DIR}/src/realsense_rgbd.cpp + ${REALSENSE_MCAP_DIR}/BuildFileDescriptorSet.cpp + ${REALSENSE_MCAP_DIR}/generated/foxglove/RawImage.pb.cc + ${REALSENSE_MCAP_DIR}/generated/foxglove/PoseInFrame.pb.cc + ${REALSENSE_MCAP_DIR}/generated/foxglove/Pose.pb.cc + ${REALSENSE_MCAP_DIR}/generated/foxglove/Quaternion.pb.cc + ${REALSENSE_MCAP_DIR}/generated/foxglove/Vector3.pb.cc + ) + target_include_directories(realsense_rgbd PRIVATE + ${realsense2_INCLUDE_DIR} + ${LIVEKIT_BRIDGE_INCLUDE} + ${LIVEKIT_INCLUDE} + ${CMAKE_CURRENT_SOURCE_DIR}/generated + ${CMAKE_CURRENT_SOURCE_DIR} + ) + target_link_directories(realsense_rgbd PRIVATE "${LIVEKIT_LIB_DIR}") + target_link_libraries(realsense_rgbd PRIVATE + ${LIVEKIT_BRIDGE_LIB} + ${LIVEKIT_LIB} + ${realsense2_LIBRARY} + protobuf::libprotobuf + ZLIB::ZLIB + ) + if(UNIX AND NOT APPLE) + # Embed SDK lib path so the loader finds liblivekit_bridge.so and liblivekit.so at runtime + set_target_properties(realsense_rgbd PROPERTIES + BUILD_RPATH "${LIVEKIT_LIB_DIR}" + INSTALL_RPATH "${LIVEKIT_LIB_DIR}" + BUILD_WITH_INSTALL_RPATH TRUE + ) + endif() + + # rgbd_viewer: subscribes to realsense_rgbd video + data track, writes MCAP + add_executable(rgbd_viewer + ${REALSENSE_MCAP_DIR}/src/rgbd_viewer.cpp + ${REALSENSE_MCAP_DIR}/BuildFileDescriptorSet.cpp + ${REALSENSE_MCAP_DIR}/generated/foxglove/RawImage.pb.cc + ) + target_include_directories(rgbd_viewer PRIVATE + ${LIVEKIT_BRIDGE_INCLUDE} + ${LIVEKIT_INCLUDE} + ${CMAKE_CURRENT_SOURCE_DIR}/external/mcap/cpp/mcap/include + ${CMAKE_CURRENT_SOURCE_DIR}/generated + ${CMAKE_CURRENT_SOURCE_DIR} + ) + target_link_directories(rgbd_viewer PRIVATE "${LIVEKIT_LIB_DIR}") + target_link_libraries(rgbd_viewer PRIVATE + ${LIVEKIT_BRIDGE_LIB} + ${LIVEKIT_LIB} + protobuf::libprotobuf + PkgConfig::ZSTD + PkgConfig::LZ4 + ZLIB::ZLIB + ) + if(UNIX AND NOT APPLE) + set_target_properties(rgbd_viewer PROPERTIES + BUILD_RPATH "${LIVEKIT_LIB_DIR}" + INSTALL_RPATH "${LIVEKIT_LIB_DIR}" + BUILD_WITH_INSTALL_RPATH TRUE + ) + endif() +message(STATUS "LiveKit participants: realsense_rgbd, rgbd_viewer (LiveKitBuild_DIR=${LiveKitBuild_DIR})") diff --git a/examples/realsense-livekit/realsense-to-mcap/README.md b/examples/realsense-livekit/realsense-to-mcap/README.md new file mode 100644 index 0000000..a54594a --- /dev/null +++ b/examples/realsense-livekit/realsense-to-mcap/README.md @@ -0,0 +1,130 @@ +# RealSense RGB-D → MCAP (Foxglove Protobuf, No ROS) + +This project records RGB and depth frames from an **Intel RealSense D415** and writes them directly to an MCAP file using Foxglove Protobuf schemas — **without using ROS**. + +The resulting `.mcap` file opens directly in Foxglove Studio with working Image panels. + +--- + +## Overview + +This example: + +- Captures RGB (`rgb8`) and depth (`16UC1`) frames +- Serializes frames using `foxglove.Image` (Protobuf) +- Writes messages to MCAP with ZSTD compression +- Produces a file immediately viewable in Foxglove + +Architecture: +RealSense D415 +↓ +librealsense2 +↓ +foxglove.Image (protobuf) +↓ +MCAP writer +↓ +realsense_rgbd.mcap + + +__NOTE: This has only been tested on Linux and MacOS.__ + +__NOTE: This has only been tested with a D415 camera.__ + +--- + +## Dependencies + +This project uses: + +- Intel RealSense D415 +- librealsense2 +- Protobuf +- MCAP C++ library +- Foxglove protobuf schemas +- Foxglove Studio + +--- + +## Install Dependencies + +### System packages + +**Ubuntu / Debian** + +```bash +sudo apt install librealsense2-dev protobuf-compiler libprotobuf-dev \ + libzstd-dev liblz4-dev zlib1g-dev +``` + +**macOS (Homebrew)** + +```bash +brew install librealsense protobuf zstd lz4 pkg-config +``` + +> zlib ships with macOS and does not need to be installed separately. + +### Clone externals and generate protobuf files + +A setup script handles cloning the MCAP C++ library and the Foxglove SDK +(for proto schemas), then runs `protoc` to generate C++ sources: + +```bash +# From anywhere — the script locates paths relative to itself +../setup_realsense.sh +``` + +This populates `external/` (mcap, foxglove-sdk) and `generated/` (protobuf +C++ sources). Both directories are gitignored. The script is idempotent and +safe to re-run. + +## Build + +All executables are built into `build/bin/`. + +1. Build the LiveKit C++ SDK (including the bridge) from the **client-sdk-cpp** repo root first (e.g. `./build.sh release`). This project links against that build. +2. From this directory: + +```bash +mkdir build +cd build +cmake .. +make -j +``` + +When this project lives under `client-sdk-cpp/examples/realsense-livekit/realsense-to-mcap/`, the SDK build directory is auto-detected (`build-release` or `build` at repo root). Otherwise set it explicitly: + +```bash +cmake -DLiveKitBuild_DIR=/path/to/client-sdk-cpp/build-release .. +``` + +### Executables (in `build/bin/`) + +| Binary | Description | +|--------|-------------| +| `realsense_to_mcap` | Standalone recorder: captures RealSense and writes RGB + depth to an MCAP file. | +| `realsense_rgbd` | LiveKit publisher: captures RealSense, publishes RGB as video and depth as DataTrack (identity `realsense_rgbd`). | +| `rgbd_viewer` | LiveKit subscriber: subscribes to that video + data track and writes to MCAP (identity `rgbd_viewer`). | + +Run the standalone recorder: + +```bash +./bin/realsense_to_mcap +``` + +This produces `realsense_rgbd.mcap` in the current directory. + +--- + +## LiveKit participants + +To stream RGB+D over a LiveKit room and record on the viewer side: + +1. Start **realsense_rgbd** (publisher), then **rgbd_viewer** (subscriber), using the same room URL and tokens with identities `realsense_rgbd` and `rgbd_viewer`. +2. Run from the build directory: + +```bash +./bin/realsense_rgbd +./bin/rgbd_viewer [output.mcap] +``` diff --git a/examples/realsense-livekit/realsense-to-mcap/src/realsense_rgbd.cpp b/examples/realsense-livekit/realsense-to-mcap/src/realsense_rgbd.cpp new file mode 100644 index 0000000..f2d4502 --- /dev/null +++ b/examples/realsense-livekit/realsense-to-mcap/src/realsense_rgbd.cpp @@ -0,0 +1,495 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* + * realsense_rgbd — LiveKit participant that captures RealSense RGB+D frames, + * publishes RGB as a video track, depth as a DataTrack (foxglove.RawImage), + * and IMU-derived orientation as a DataTrack (foxglove.PoseInFrame). + * + * If the RealSense device has an IMU (e.g. D435i), gyroscope and + * accelerometer data are fused via a complementary filter to produce a + * camera/pose topic. Devices without an IMU skip pose publishing. + * + * Usage: + * realsense_rgbd + * LIVEKIT_URL=... LIVEKIT_TOKEN=... realsense_rgbd + * + * Token must grant identity "realsense_rgbd". Run rgbd_viewer in the same room + * to receive and record to MCAP. + */ + +#include "livekit_bridge/bridge_data_track.h" +#include "livekit_bridge/livekit_bridge.h" +#include "livekit/track.h" + +#include "BuildFileDescriptorSet.h" +#include "foxglove/PoseInFrame.pb.h" +#include "foxglove/RawImage.pb.h" + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static volatile std::sig_atomic_t g_running = 1; +static void signalHandler(int) { g_running = 0; } + +static uint64_t nowNs() { + return std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); +} + +static std::string nowStr() { + auto now = std::chrono::system_clock::now(); + auto ms = std::chrono::duration_cast( + now.time_since_epoch()) % 1000; + std::time_t t = std::chrono::system_clock::to_time_t(now); + std::tm tm{}; + localtime_r(&t, &tm); + std::ostringstream os; + os << std::put_time(&tm, "%H:%M:%S") << '.' << std::setfill('0') + << std::setw(3) << ms.count(); + return os.str(); +} + +struct OrientationQuat { + double x, y, z, w; +}; + +/// Convert intrinsic XYZ Euler angles (radians) to a unit quaternion. +static OrientationQuat eulerToQuaternion(double x, double y, double z) { + double cx = std::cos(x / 2), sx = std::sin(x / 2); + double cy = std::cos(y / 2), sy = std::sin(y / 2); + double cz = std::cos(z / 2), sz = std::sin(z / 2); + return { + sx * cy * cz + cx * sy * sz, + cx * sy * cz - sx * cy * sz, + cx * cy * sz + sx * sy * cz, + cx * cy * cz - sx * sy * sz, + }; +} + +/// Fuse gyroscope + accelerometer into an orientation estimate using a +/// complementary filter. Adapted from the librealsense motion example. +class RotationEstimator { +public: + void process_gyro(rs2_vector gyro_data, double ts) { + if (first_gyro_) { + first_gyro_ = false; + last_ts_gyro_ = ts; + return; + } + float dt = static_cast((ts - last_ts_gyro_) / 1000.0); + last_ts_gyro_ = ts; + + std::lock_guard lock(mtx_); + theta_x_ += -gyro_data.z * dt; + theta_y_ += -gyro_data.y * dt; + theta_z_ += gyro_data.x * dt; + } + + void process_accel(rs2_vector accel_data) { + float accel_angle_x = std::atan2(accel_data.x, + std::sqrt(accel_data.y * accel_data.y + + accel_data.z * accel_data.z)); + float accel_angle_z = std::atan2(accel_data.y, accel_data.z); + + std::lock_guard lock(mtx_); + if (first_accel_) { + first_accel_ = false; + theta_x_ = accel_angle_x; + theta_y_ = static_cast(M_PI); + theta_z_ = accel_angle_z; + return; + } + theta_x_ = theta_x_ * kAlpha + accel_angle_x * (1.0f - kAlpha); + theta_z_ = theta_z_ * kAlpha + accel_angle_z * (1.0f - kAlpha); + } + + OrientationQuat get_orientation() const { + std::lock_guard lock(mtx_); + return eulerToQuaternion( + static_cast(theta_x_), + static_cast(theta_y_), + static_cast(theta_z_)); + } + +private: + static constexpr float kAlpha = 0.98f; + mutable std::mutex mtx_; + float theta_x_ = 0, theta_y_ = 0, theta_z_ = 0; + bool first_gyro_ = true; + bool first_accel_ = true; + double last_ts_gyro_ = 0; +}; + +/// Convert RGB8 to RGBA (alpha = 0xFF). Assumes dst has size width*height*4. +static void rgb8ToRgba(const std::uint8_t* rgb, std::uint8_t* rgba, + int width, int height) { + const int rgbStep = width * 3; + const int rgbaStep = width * 4; + for (int y = 0; y < height; ++y) { + const std::uint8_t* src = rgb + y * rgbStep; + std::uint8_t* dst = rgba + y * rgbaStep; + for (int x = 0; x < width; ++x) { + *dst++ = *src++; + *dst++ = *src++; + *dst++ = *src++; + *dst++ = 0xFF; + } + } +} + +int main(int argc, char* argv[]) { + GOOGLE_PROTOBUF_VERIFY_VERSION; + + std::signal(SIGINT, signalHandler); + std::signal(SIGTERM, signalHandler); + + std::string url; + std::string token; + const char* env_url = std::getenv("LIVEKIT_URL"); + const char* env_token = std::getenv("LIVEKIT_TOKEN"); + if (argc >= 3) { + url = argv[1]; + token = argv[2]; + } else if (env_url && env_token) { + url = env_url; + token = env_token; + } else { + std::cerr << "Usage: realsense_rgbd \n" + " or: LIVEKIT_URL=... LIVEKIT_TOKEN=... realsense_rgbd\n"; + return 1; + } + + const int kWidth = 640; + const int kHeight = 480; + const int kDepthFps = 10; // data track depth rate (Hz); limited by SCTP throughput + const int kPoseFps = 30; + + RotationEstimator rotation_est; + + // Check for IMU support (following rs-motion.cpp pattern). + bool imu_supported = false; + { + rs2::context ctx; + auto devices = ctx.query_devices(); + for (auto dev : devices) { + bool found_gyro = false, found_accel = false; + for (auto sensor : dev.query_sensors()) { + for (auto profile : sensor.get_stream_profiles()) { + if (profile.stream_type() == RS2_STREAM_GYRO) + found_gyro = true; + if (profile.stream_type() == RS2_STREAM_ACCEL) + found_accel = true; + } + } + if (found_gyro && found_accel) { + imu_supported = true; + break; + } + } + } + + // Color+depth pipeline. + rs2::pipeline pipe; + rs2::config cfg; + cfg.enable_stream(RS2_STREAM_COLOR, kWidth, kHeight, RS2_FORMAT_RGB8, 30); + cfg.enable_stream(RS2_STREAM_DEPTH, kWidth, kHeight, RS2_FORMAT_Z16, 30); + try { + pipe.start(cfg); + } catch (const rs2::error& e) { + std::cerr << "RealSense error: " << e.what() << "\n"; + return 1; + } + + // Separate IMU-only pipeline with callback, mirroring rs-motion.cpp. + // A dedicated pipeline for gyro+accel avoids interfering with the + // color+depth pipeline's frame syncer. + rs2::pipeline imu_pipe; + bool has_imu = false; + if (imu_supported) { + rs2::config imu_cfg; + imu_cfg.enable_stream(RS2_STREAM_ACCEL, RS2_FORMAT_MOTION_XYZ32F); + imu_cfg.enable_stream(RS2_STREAM_GYRO, RS2_FORMAT_MOTION_XYZ32F); + try { + imu_pipe.start(imu_cfg, [&rotation_est](rs2::frame frame) { + auto motion = frame.as(); + if (motion && + motion.get_profile().stream_type() == RS2_STREAM_GYRO && + motion.get_profile().format() == RS2_FORMAT_MOTION_XYZ32F) { + rotation_est.process_gyro(motion.get_motion_data(), + motion.get_timestamp()); + } + if (motion && + motion.get_profile().stream_type() == RS2_STREAM_ACCEL && + motion.get_profile().format() == RS2_FORMAT_MOTION_XYZ32F) { + rotation_est.process_accel(motion.get_motion_data()); + } + }); + has_imu = true; + std::cout << "[realsense_rgbd] IMU pipeline started.\n"; + } catch (const rs2::error& e) { + std::cerr << "[realsense_rgbd] Could not start IMU pipeline: " + << e.what() << " — continuing without pose.\n"; + } + } else { + std::cout << "[realsense_rgbd] IMU not available, continuing without " + "pose.\n"; + } + + livekit_bridge::LiveKitBridge bridge; + livekit::RoomOptions options; + options.auto_subscribe = false; + + std::cout << "[realsense_rgbd] Connecting to " << url << " ...\n"; + if (!bridge.connect(url, token, options)) { + std::cerr << "[realsense_rgbd] Failed to connect.\n"; + pipe.stop(); + return 1; + } + + std::shared_ptr video_track; + std::shared_ptr depth_track; + std::shared_ptr pose_track; + std::shared_ptr hello_track; + + try { + video_track = bridge.createVideoTrack("camera/color", kWidth, kHeight, + livekit::TrackSource::SOURCE_CAMERA); + depth_track = bridge.createDataTrack("camera/depth"); + if (has_imu) { + pose_track = bridge.createDataTrack("camera/pose"); + } + hello_track = bridge.createDataTrack("hello"); + } catch (const std::exception& e) { + std::cerr << "[realsense_rgbd] Failed to create tracks: " << e.what() + << "\n"; + bridge.disconnect(); + pipe.stop(); + return 1; + } + + std::cout << "[realsense_rgbd] Publishing camera/color (video), " + "camera/depth (DataTrack)" + << (has_imu ? ", camera/pose (DataTrack)" : "") + << ", and hello (DataTrack). Press Ctrl+C to stop.\n"; + + std::vector rgbaBuf(static_cast(kWidth * kHeight * 4)); + + uint32_t hello_seq = 0; + uint32_t depth_pushed = 0; + uint32_t pose_pushed = 0; + uint32_t last_depth_report_count = 0; + auto last_hello = std::chrono::steady_clock::now(); + auto last_depth = std::chrono::steady_clock::now(); + auto last_pose = std::chrono::steady_clock::now(); + auto last_depth_report = std::chrono::steady_clock::now(); + const auto depth_interval = + std::chrono::microseconds(1000000 / kDepthFps); + const auto pose_interval = + std::chrono::microseconds(1000000 / kPoseFps); + + constexpr auto kMinLoopDuration = std::chrono::milliseconds(15); + + while (g_running) { + auto loop_start = std::chrono::steady_clock::now(); + + // Periodic "hello viewer" test message every 10 seconds + if (loop_start - last_hello >= std::chrono::seconds(1)) { + last_hello = loop_start; + ++hello_seq; + std::string text = "hello viewer #" + std::to_string(hello_seq); + uint64_t ts_us = static_cast(nowNs() / 1000); + bool ok = hello_track->pushFrame( + reinterpret_cast(text.data()), text.size(), + ts_us); + std::cout << "[" << nowStr() << "] [realsense_rgbd] Sent hello #" + << hello_seq << " (" << text.size() << " bytes) -> " + << (ok ? "ok" : "FAILED") << "\n"; + } + + rs2::frameset frames; + if (!pipe.poll_for_frames(&frames)) { + auto elapsed = std::chrono::steady_clock::now() - loop_start; + if (elapsed < kMinLoopDuration) + std::this_thread::sleep_for(kMinLoopDuration - elapsed); + continue; + } + + auto color = frames.get_color_frame(); + auto depth = frames.get_depth_frame(); + if (!color || !depth) { + auto elapsed = std::chrono::steady_clock::now() - loop_start; + if (elapsed < kMinLoopDuration) + std::this_thread::sleep_for(kMinLoopDuration - elapsed); + continue; + } + + const uint64_t timestamp_ns = nowNs(); + const std::int64_t timestamp_us = + static_cast(timestamp_ns / 1000); + const int64_t secs = static_cast(timestamp_ns / 1000000000ULL); + const int32_t nsecs = static_cast(timestamp_ns % 1000000000ULL); + + // RGB → RGBA and push to video track + rgb8ToRgba(static_cast(color.get_data()), + rgbaBuf.data(), kWidth, kHeight); + if (!video_track->pushFrame(rgbaBuf.data(), rgbaBuf.size(), timestamp_us)) { + break; + } + + // Depth as RawImage proto on DataTrack (throttled to kDepthFps) + if (loop_start - last_depth >= depth_interval) { + last_depth = loop_start; + + foxglove::RawImage msg; + auto* ts = msg.mutable_timestamp(); + ts->set_seconds(secs); + ts->set_nanos(nsecs); + msg.set_frame_id("camera_depth"); + msg.set_width(depth.get_width()); + msg.set_height(depth.get_height()); + msg.set_encoding("16UC1"); + msg.set_step(depth.get_width() * 2); + msg.set_data(depth.get_data(), depth.get_data_size()); + + std::string serialized = msg.SerializeAsString(); + + uLongf comp_bound = compressBound(static_cast(serialized.size())); + std::vector compressed(comp_bound); + uLongf comp_size = comp_bound; + int zrc = compress2( + compressed.data(), &comp_size, + reinterpret_cast(serialized.data()), + static_cast(serialized.size()), Z_BEST_SPEED); + + auto push_start = std::chrono::steady_clock::now(); + bool ok = false; + if (zrc == Z_OK) { + ok = depth_track->pushFrame( + compressed.data(), static_cast(comp_size), + static_cast(timestamp_us)); + } else { + std::cerr << "[realsense_rgbd] zlib compress failed (" << zrc + << "), sending uncompressed\n"; + ok = depth_track->pushFrame( + reinterpret_cast(serialized.data()), + serialized.size(), static_cast(timestamp_us)); + } + auto push_dur = std::chrono::steady_clock::now() - push_start; + double push_ms = + std::chrono::duration_cast(push_dur) + .count() / 1000.0; + + ++depth_pushed; + if (!ok) { + std::cout << "[" << nowStr() + << "] [realsense_rgbd] Failed to push depth frame #" + << depth_pushed << " (push took " << std::fixed + << std::setprecision(1) << push_ms << "ms)\n"; + break; + } + if (depth_pushed == 1 || depth_pushed % 10 == 0) { + double elapsed_sec = + std::chrono::duration_cast( + loop_start - last_depth_report) + .count() / 1000.0; + double actual_fps = + (elapsed_sec > 0) + ? static_cast(depth_pushed - last_depth_report_count) / + elapsed_sec + : 0; + std::cout << "[" << nowStr() + << "] [realsense_rgbd] Depth #" << depth_pushed + << " push=" << std::fixed << std::setprecision(1) << push_ms + << "ms " << serialized.size() << "B->" + << (zrc == Z_OK ? comp_size : serialized.size()) << "B" + << " actual=" << std::setprecision(1) << actual_fps + << "fps\n"; + last_depth_report = loop_start; + last_depth_report_count = depth_pushed; + } + } + + // Pose as PoseInFrame proto on DataTrack (throttled to kPoseFps) + if (has_imu && pose_track && (loop_start - last_pose >= pose_interval)) { + last_pose = loop_start; + + auto orientation = rotation_est.get_orientation(); + + foxglove::PoseInFrame pose_msg; + auto* pose_ts = pose_msg.mutable_timestamp(); + pose_ts->set_seconds(secs); + pose_ts->set_nanos(nsecs); + pose_msg.set_frame_id("camera_imu"); + + auto* pose = pose_msg.mutable_pose(); + auto* pos = pose->mutable_position(); + pos->set_x(0); + pos->set_y(0); + pos->set_z(0); + auto* orient = pose->mutable_orientation(); + orient->set_x(orientation.x); + orient->set_y(orientation.y); + orient->set_z(orientation.z); + orient->set_w(orientation.w); + + std::string pose_serialized = pose_msg.SerializeAsString(); + bool pose_ok = pose_track->pushFrame( + reinterpret_cast(pose_serialized.data()), + pose_serialized.size(), static_cast(timestamp_us)); + ++pose_pushed; + if (!pose_ok) { + std::cout << "[" << nowStr() + << "] [realsense_rgbd] Failed to push pose frame #" + << pose_pushed << "\n"; + } + if (pose_pushed == 1 || pose_pushed % 100 == 0) { + std::cout << "[" << nowStr() + << "] [realsense_rgbd] Pose #" << pose_pushed + << " " << pose_serialized.size() << "B" + << " q=(" << std::fixed << std::setprecision(3) + << orientation.x << ", " << orientation.y << ", " + << orientation.z << ", " << orientation.w << ")\n"; + } + } + + auto elapsed = std::chrono::steady_clock::now() - loop_start; + if (elapsed < kMinLoopDuration) { + std::this_thread::sleep_for(kMinLoopDuration - elapsed); + } + } + + std::cout << "[realsense_rgbd] Stopping...\n"; + bridge.disconnect(); + if (has_imu) imu_pipe.stop(); + pipe.stop(); + google::protobuf::ShutdownProtobufLibrary(); + return 0; +} diff --git a/examples/realsense-livekit/realsense-to-mcap/src/realsense_to_mcap.cpp b/examples/realsense-livekit/realsense-to-mcap/src/realsense_to_mcap.cpp new file mode 100644 index 0000000..1d7b2b2 --- /dev/null +++ b/examples/realsense-livekit/realsense-to-mcap/src/realsense_to_mcap.cpp @@ -0,0 +1,159 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#define MCAP_IMPLEMENTATION +#include + +#include + +#include "BuildFileDescriptorSet.h" +#include "foxglove/RawImage.pb.h" + +#include +#include +#include + +static volatile std::sig_atomic_t g_running = 1; + +static void signalHandler(int signum) { + (void)signum; + g_running = 0; +} + +static uint64_t nowNs() { + return std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); +} + +int main() { + GOOGLE_PROTOBUF_VERIFY_VERSION; + + std::signal(SIGINT, signalHandler); + std::signal(SIGTERM, signalHandler); + + rs2::pipeline pipe; + rs2::config cfg; + cfg.enable_stream(RS2_STREAM_COLOR, 640, 480, RS2_FORMAT_RGB8, 30); + cfg.enable_stream(RS2_STREAM_DEPTH, 640, 480, RS2_FORMAT_Z16, 30); + pipe.start(cfg); + + mcap::McapWriter writer; + auto options = mcap::McapWriterOptions(""); + options.compression = mcap::Compression::Zstd; + + { + const auto res = writer.open("realsense_rgbd.mcap", options); + if (!res.ok()) { + std::cerr << "Failed to open MCAP file: " << res.message << std::endl; + return 1; + } + } + + // Register schema as a serialized FileDescriptorSet (required by MCAP protobuf profile) + mcap::Schema schema( + "foxglove.RawImage", "protobuf", + BuildFileDescriptorSet(foxglove::RawImage::descriptor()).SerializeAsString()); + writer.addSchema(schema); + + mcap::Channel colorChannel("camera/color", "protobuf", schema.id); + writer.addChannel(colorChannel); + + mcap::Channel depthChannel("camera/depth", "protobuf", schema.id); + writer.addChannel(depthChannel); + + std::cout << "Recording to realsense_rgbd.mcap ... Press Ctrl+C to stop.\n"; + + uint32_t seq = 0; + while (g_running) { + rs2::frameset frames; + if (!pipe.poll_for_frames(&frames)) { + continue; + } + + auto color = frames.get_color_frame(); + auto depth = frames.get_depth_frame(); + if (!color || !depth) { + continue; + } + + uint64_t timestamp = nowNs(); + int64_t secs = static_cast(timestamp / 1000000000ULL); + int32_t nsecs = static_cast(timestamp % 1000000000ULL); + + // Color image + { + foxglove::RawImage msg; + auto* ts = msg.mutable_timestamp(); + ts->set_seconds(secs); + ts->set_nanos(nsecs); + msg.set_frame_id("camera_color"); + msg.set_width(color.get_width()); + msg.set_height(color.get_height()); + msg.set_encoding("rgb8"); + msg.set_step(color.get_width() * 3); + msg.set_data(color.get_data(), color.get_data_size()); + + std::string serialized = msg.SerializeAsString(); + + mcap::Message mcapMsg; + mcapMsg.channelId = colorChannel.id; + mcapMsg.sequence = seq; + mcapMsg.logTime = timestamp; + mcapMsg.publishTime = timestamp; + mcapMsg.data = reinterpret_cast(serialized.data()); + mcapMsg.dataSize = serialized.size(); + const auto res = writer.write(mcapMsg); + if (!res.ok()) { + std::cerr << "Failed to write color message: " << res.message << std::endl; + } + } + + // Depth image + { + foxglove::RawImage msg; + auto* ts = msg.mutable_timestamp(); + ts->set_seconds(secs); + ts->set_nanos(nsecs); + msg.set_frame_id("camera_depth"); + msg.set_width(depth.get_width()); + msg.set_height(depth.get_height()); + msg.set_encoding("16UC1"); + msg.set_step(depth.get_width() * 2); + msg.set_data(depth.get_data(), depth.get_data_size()); + + std::string serialized = msg.SerializeAsString(); + + mcap::Message mcapMsg; + mcapMsg.channelId = depthChannel.id; + mcapMsg.sequence = seq; + mcapMsg.logTime = timestamp; + mcapMsg.publishTime = timestamp; + mcapMsg.data = reinterpret_cast(serialized.data()); + mcapMsg.dataSize = serialized.size(); + const auto res = writer.write(mcapMsg); + if (!res.ok()) { + std::cerr << "Failed to write depth message: " << res.message << std::endl; + } + } + + ++seq; + } + + std::cout << "\nStopping... wrote " << seq << " frame pairs.\n"; + writer.close(); + pipe.stop(); + google::protobuf::ShutdownProtobufLibrary(); + return 0; +} diff --git a/examples/realsense-livekit/realsense-to-mcap/src/rgbd_viewer.cpp b/examples/realsense-livekit/realsense-to-mcap/src/rgbd_viewer.cpp new file mode 100644 index 0000000..25df34e --- /dev/null +++ b/examples/realsense-livekit/realsense-to-mcap/src/rgbd_viewer.cpp @@ -0,0 +1,477 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* + * rgbd_viewer — LiveKit participant that subscribes to realsense_rgbd's + * video track (RGB) and data track (depth), and writes both to an MCAP file. + * + * Usage: + * rgbd_viewer [output.mcap] + * LIVEKIT_URL=... LIVEKIT_TOKEN=... rgbd_viewer [output.mcap] + * + * Token must grant identity "rgbd_viewer". Start realsense_rgbd in the same + * room first to publish camera/color and camera/depth. + */ + +#define MCAP_IMPLEMENTATION +#include + +#include "livekit/track.h" +#include "livekit/video_frame.h" +#include "livekit_bridge/livekit_bridge.h" + +#include "BuildFileDescriptorSet.h" +#include "foxglove/RawImage.pb.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static volatile std::sig_atomic_t g_running = 1; +static void signalHandler(int) { g_running = 0; } + +static std::string nowStr() { + auto now = std::chrono::system_clock::now(); + auto ms = std::chrono::duration_cast( + now.time_since_epoch()) % + 1000; + std::time_t t = std::chrono::system_clock::to_time_t(now); + std::tm tm{}; + localtime_r(&t, &tm); + std::ostringstream os; + os << std::put_time(&tm, "%H:%M:%S") << '.' << std::setfill('0') + << std::setw(3) << ms.count(); + return os.str(); +} + +static const char kSenderIdentity[] = "realsense_rgbd"; +static const char kColorTrackName[] = "camera/color"; // video track +static const char kDepthTrackName[] = "camera/depth"; // data track +static const char kHelloTrackName[] = "hello"; // test data track + +/// MCAP writer with a background write thread. +/// +/// All public write methods enqueue entries and return immediately so reader +/// callbacks are never blocked by Zstd compression or disk I/O. A dedicated +/// writer thread drains the queue and performs the actual MCAP writes. +struct McapRecorder { + mcap::McapWriter writer; + mcap::ChannelId colorChannelId = 0; + mcap::ChannelId depthChannelId = 0; + mcap::ChannelId helloChannelId = 0; + uint32_t colorSeq = 0; + uint32_t depthSeq = 0; + uint32_t helloSeq = 0; + bool open = false; + + struct WriteEntry { + enum Type { kColor, kDepth, kHello } type; + std::vector data; + std::string text; + int width = 0; + int height = 0; + std::optional user_timestamp_us; + uint64_t wall_time_ns = 0; + }; + + std::deque queue_; + std::mutex queue_mtx_; + std::condition_variable queue_cv_; + std::thread writer_thread_; + std::atomic stop_{false}; + std::atomic color_enqueue_seq_{0}; + + std::atomic depth_received{0}; + std::atomic hello_received{0}; + std::atomic last_depth_latency_us{-1}; + std::atomic last_hello_latency_us{-1}; + + static constexpr size_t kMaxQueueSize = 60; + + static uint64_t wallNs() { + return std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + } + + bool openFile(const std::string &path) { + mcap::McapWriterOptions opts(""); + opts.compression = mcap::Compression::Zstd; + auto res = writer.open(path, opts); + if (!res.ok()) { + std::cerr << "[rgbd_viewer] Failed to open MCAP: " << res.message << "\n"; + return false; + } + + mcap::Schema rawImageSchema( + "foxglove.RawImage", "protobuf", + BuildFileDescriptorSet(foxglove::RawImage::descriptor()) + .SerializeAsString()); + writer.addSchema(rawImageSchema); + + mcap::Schema textSchema("hello", "jsonschema", + R"({"type":"object","properties":{"text":{"type":"string"}}})"); + writer.addSchema(textSchema); + + mcap::Channel colorChannel("camera/color", "protobuf", rawImageSchema.id); + writer.addChannel(colorChannel); + colorChannelId = colorChannel.id; + + mcap::Channel depthChannel("camera/depth", "protobuf", rawImageSchema.id); + writer.addChannel(depthChannel); + depthChannelId = depthChannel.id; + + mcap::Channel helloChannel("hello", "json", textSchema.id); + writer.addChannel(helloChannel); + helloChannelId = helloChannel.id; + + open = true; + writer_thread_ = std::thread([this] { writerLoop(); }); + return true; + } + + // Throttle color to ~10fps to reduce memory/write pressure (video arrives + // at 30fps but each RGBA frame is ~1.2 MB before Zstd). + void writeColorFrame(const std::uint8_t *rgba, std::size_t size, int width, + int height, std::int64_t /*timestamp_us*/) { + uint32_t seq = color_enqueue_seq_.fetch_add(1, std::memory_order_relaxed); + if (seq % 3 != 0) return; + + WriteEntry e; + e.type = WriteEntry::kColor; + e.data.assign(rgba, rgba + size); + e.width = width; + e.height = height; + e.wall_time_ns = wallNs(); + enqueue(std::move(e)); + } + + void writeDepthPayload(const std::uint8_t *data, std::size_t size, + std::optional user_timestamp_us) { + WriteEntry e; + e.type = WriteEntry::kDepth; + e.data.assign(data, data + size); + e.user_timestamp_us = user_timestamp_us; + e.wall_time_ns = wallNs(); + enqueue(std::move(e)); + } + + void writeHello(const std::string &text, + std::optional user_timestamp_us) { + WriteEntry e; + e.type = WriteEntry::kHello; + e.text = text; + e.user_timestamp_us = user_timestamp_us; + e.wall_time_ns = wallNs(); + enqueue(std::move(e)); + } + + void close() { + stop_.store(true, std::memory_order_release); + queue_cv_.notify_one(); + if (writer_thread_.joinable()) + writer_thread_.join(); + if (open) { + writer.close(); + open = false; + } + } + +private: + void enqueue(WriteEntry &&e) { + std::lock_guard lock(queue_mtx_); + if (queue_.size() >= kMaxQueueSize) { + for (auto it = queue_.begin(); it != queue_.end(); ++it) { + if (it->type == WriteEntry::kColor) { + queue_.erase(it); + break; + } + } + } + queue_.push_back(std::move(e)); + } + + void writerLoop() { + while (true) { + { + std::unique_lock lock(queue_mtx_); + queue_cv_.wait_for(lock, std::chrono::seconds(1), [this] { + return stop_.load(std::memory_order_acquire); + }); + } + + std::deque batch; + { + std::lock_guard lock(queue_mtx_); + batch.swap(queue_); + } + + uint32_t nc = 0, nd = 0, nh = 0; + for (auto &e : batch) { + switch (e.type) { + case WriteEntry::kColor: + flushColor(e); + ++nc; + break; + case WriteEntry::kDepth: + flushDepth(e); + ++nd; + break; + case WriteEntry::kHello: + flushHello(e); + ++nh; + break; + } + } + + if (nc + nd + nh > 0) { + auto cr = color_enqueue_seq_.load(std::memory_order_relaxed); + auto dr = depth_received.load(std::memory_order_relaxed); + auto hr = hello_received.load(std::memory_order_relaxed); + auto dl = last_depth_latency_us.load(std::memory_order_relaxed); + auto hl = last_hello_latency_us.load(std::memory_order_relaxed); + std::cout << "[" << nowStr() << "] [rgbd_viewer] wrote " + << nc << "c " << nd << "d " << nh << "h" + << " | totals " << cr << "c " << dr << "d " << hr << "h"; + if (dl >= 0) + std::cout << " | depth_lat=" << std::fixed << std::setprecision(1) + << dl / 1000.0 << "ms"; + if (hl >= 0) + std::cout << " | hello_lat=" << std::fixed << std::setprecision(1) + << hl / 1000.0 << "ms"; + std::cout << "\n"; + } + + if (stop_.load(std::memory_order_acquire) && batch.empty()) + break; + } + } + + void flushColor(const WriteEntry &e) { + foxglove::RawImage msg; + auto *ts = msg.mutable_timestamp(); + ts->set_seconds(static_cast(e.wall_time_ns / 1000000000ULL)); + ts->set_nanos(static_cast(e.wall_time_ns % 1000000000ULL)); + msg.set_frame_id("camera_color"); + msg.set_width(e.width); + msg.set_height(e.height); + msg.set_encoding("rgba8"); + msg.set_step(e.width * 4); + msg.set_data(e.data.data(), e.data.size()); + + std::string serialized = msg.SerializeAsString(); + mcap::Message m; + m.channelId = colorChannelId; + m.sequence = colorSeq++; + m.logTime = e.wall_time_ns; + m.publishTime = e.wall_time_ns; + m.data = reinterpret_cast(serialized.data()); + m.dataSize = serialized.size(); + auto res = writer.write(m); + if (!res.ok()) { + std::cerr << "[rgbd_viewer] Write color error: " << res.message << "\n"; + } + } + + void flushDepth(const WriteEntry &e) { + const uint64_t publishTime = + (e.user_timestamp_us && *e.user_timestamp_us > 0) + ? *e.user_timestamp_us * 1000ULL + : e.wall_time_ns; + + // Depth payloads arrive zlib-compressed from the sender. Decompress + // so the MCAP file contains standard foxglove.RawImage protobuf bytes. + const std::uint8_t *write_ptr = e.data.data(); + std::size_t write_size = e.data.size(); + + // 640*480*2 (depth) + proto overhead ≈ 620 KB; 1 MB is a safe upper bound. + static constexpr uLongf kMaxDecompressed = 1024 * 1024; + std::vector decompressed(kMaxDecompressed); + uLongf decompressed_size = kMaxDecompressed; + int zrc = uncompress(decompressed.data(), &decompressed_size, + e.data.data(), + static_cast(e.data.size())); + if (zrc == Z_OK) { + write_ptr = decompressed.data(); + write_size = static_cast(decompressed_size); + } + + mcap::Message m; + m.channelId = depthChannelId; + m.sequence = depthSeq++; + m.logTime = e.wall_time_ns; + m.publishTime = publishTime; + m.data = reinterpret_cast(write_ptr); + m.dataSize = write_size; + auto res = writer.write(m); + if (!res.ok()) { + std::cerr << "[rgbd_viewer] Write depth error: " << res.message << "\n"; + } + } + + void flushHello(const WriteEntry &e) { + const uint64_t publishTime = + (e.user_timestamp_us && *e.user_timestamp_us > 0) + ? *e.user_timestamp_us * 1000ULL + : e.wall_time_ns; + + std::string json = "{\"text\":\"" + e.text + "\"}"; + mcap::Message m; + m.channelId = helloChannelId; + m.sequence = helloSeq++; + m.logTime = e.wall_time_ns; + m.publishTime = publishTime; + m.data = reinterpret_cast(json.data()); + m.dataSize = json.size(); + auto res = writer.write(m); + if (!res.ok()) { + std::cerr << "[rgbd_viewer] Write hello error: " << res.message << "\n"; + } + } +}; + +int main(int argc, char *argv[]) { + GOOGLE_PROTOBUF_VERIFY_VERSION; + + std::signal(SIGINT, signalHandler); + std::signal(SIGTERM, signalHandler); + + std::string outputPath = "rgbd_viewer_output.mcap"; + std::string url; + std::string token; + + const char *env_url = std::getenv("LIVEKIT_URL"); + const char *env_token = std::getenv("LIVEKIT_TOKEN"); + + if (argc >= 4) { + outputPath = argv[1]; + url = argv[2]; + token = argv[3]; + } else if (argc >= 3) { + url = argv[1]; + token = argv[2]; + } else if (env_url && env_token) { + url = env_url; + token = env_token; + if (argc >= 2) { + outputPath = argv[1]; + } + } else { + std::cerr << "Usage: rgbd_viewer [output.mcap] \n" + " or: LIVEKIT_URL=... LIVEKIT_TOKEN=... rgbd_viewer " + "[output.mcap]\n"; + return 1; + } + + std::shared_ptr recorder = std::make_shared(); + if (!recorder->openFile(outputPath)) { + return 1; + } + std::cout << "[rgbd_viewer] Recording to " << outputPath << "\n"; + + livekit_bridge::LiveKitBridge bridge; + + // Video callback: realsense_rgbd's camera (RGB stream as video track) + bridge.setOnVideoFrameCallback( + kSenderIdentity, livekit::TrackSource::SOURCE_CAMERA, + [recorder](const livekit::VideoFrame &frame, std::int64_t timestamp_us) { + const std::uint8_t *data = frame.data(); + const std::size_t size = frame.dataSize(); + if (data && size > 0) { + recorder->writeColorFrame(data, size, frame.width(), frame.height(), + timestamp_us); + } + }); + + // Data callback: realsense_rgbd's camera/depth (RawImage proto bytes) + bridge.setOnDataFrameCallback( + kSenderIdentity, kDepthTrackName, + [recorder](const std::vector &payload, + std::optional user_timestamp) { + if (payload.empty()) + return; + recorder->depth_received.fetch_add(1, std::memory_order_relaxed); + if (user_timestamp) { + uint64_t recv_us = + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + recorder->last_depth_latency_us.store( + static_cast(recv_us - *user_timestamp), + std::memory_order_relaxed); + } + recorder->writeDepthPayload(payload.data(), payload.size(), + user_timestamp); + }); + + // Test callback: realsense_rgbd's "hello" track (plain-text ping) + bridge.setOnDataFrameCallback( + kSenderIdentity, kHelloTrackName, + [recorder](const std::vector &payload, + std::optional user_timestamp) { + recorder->hello_received.fetch_add(1, std::memory_order_relaxed); + if (user_timestamp) { + uint64_t recv_us = + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + recorder->last_hello_latency_us.store( + static_cast(recv_us - *user_timestamp), + std::memory_order_relaxed); + } + std::string text(payload.begin(), payload.end()); + recorder->writeHello(text, user_timestamp); + }); + + std::cout << "[rgbd_viewer] Connecting to " << url << " ...\n"; + livekit::RoomOptions options; + options.auto_subscribe = true; + if (!bridge.connect(url, token, options)) { + std::cerr << "[rgbd_viewer] Failed to connect.\n"; + recorder->close(); + return 1; + } + + std::cout << "[rgbd_viewer] Connected. Waiting for " << kSenderIdentity + << " (camera/color + camera/depth). Press Ctrl+C to stop.\n"; + + while (g_running && bridge.isConnected()) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + + std::cout << "[rgbd_viewer] Stopping... (depth: " + << recorder->depth_received.load(std::memory_order_relaxed) + << ", hello: " + << recorder->hello_received.load(std::memory_order_relaxed) + << ")\n"; + bridge.disconnect(); + recorder->close(); + google::protobuf::ShutdownProtobufLibrary(); + return 0; +} diff --git a/examples/realsense-livekit/setup_realsense.sh b/examples/realsense-livekit/setup_realsense.sh new file mode 100755 index 0000000..76bfc3f --- /dev/null +++ b/examples/realsense-livekit/setup_realsense.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +# Copyright 2026 LiveKit, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Clones external dependencies and generates protobuf C++ files for the +# realsense-to-mcap example. Safe to re-run (idempotent). + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="${SCRIPT_DIR}/realsense-to-mcap" +EXTERNAL_DIR="${PROJECT_DIR}/external" +GENERATED_DIR="${PROJECT_DIR}/generated" + +MCAP_REPO="https://github.com/foxglove/mcap.git" +FOXGLOVE_SDK_REPO="https://github.com/foxglove/foxglove-sdk.git" + +# --------------------------------------------------------------------------- +# Detect platform +# --------------------------------------------------------------------------- +OS="$(uname -s)" +case "${OS}" in + Linux*) PLATFORM=linux ;; + Darwin*) PLATFORM=macos ;; + *) PLATFORM=unknown ;; +esac + +# --------------------------------------------------------------------------- +# Check system dependencies +# --------------------------------------------------------------------------- +check_cmd() { + if ! command -v "$1" &>/dev/null; then + echo "WARNING: '$1' not found. $2" + return 1 + fi + return 0 +} + +hint_install() { + local pkg_apt="$1" + local pkg_brew="$2" + if [ "${PLATFORM}" = "macos" ]; then + echo "Install via: brew install ${pkg_brew}" + else + echo "Install via: sudo apt install ${pkg_apt}" + fi +} + +missing=0 +check_cmd protoc "$(hint_install protobuf-compiler protobuf)" || missing=1 +check_cmd pkg-config "$(hint_install pkg-config pkg-config)" || missing=1 + +if pkg-config --exists realsense2 2>/dev/null; then + echo " realsense2 ... found" +else + echo "WARNING: librealsense2 not found via pkg-config." + hint_install librealsense2-dev librealsense + missing=1 +fi + +if [ "$missing" -ne 0 ]; then + echo "" + echo "Some dependencies are missing (see warnings above). You can still" + echo "continue, but the build may fail." + echo "" +fi + +# --------------------------------------------------------------------------- +# Clone / update external repos +# --------------------------------------------------------------------------- +mkdir -p "${EXTERNAL_DIR}" + +clone_or_pull() { + local repo_url="$1" + local dest="$2" + + if [ -d "${dest}/.git" ]; then + echo "Updating $(basename "${dest}") ..." + git -C "${dest}" pull --ff-only + else + echo "Cloning $(basename "${dest}") ..." + git clone --depth 1 "${repo_url}" "${dest}" + fi +} + +clone_or_pull "${MCAP_REPO}" "${EXTERNAL_DIR}/mcap" +clone_or_pull "${FOXGLOVE_SDK_REPO}" "${EXTERNAL_DIR}/foxglove-sdk" + +# --------------------------------------------------------------------------- +# Generate C++ protobuf files from Foxglove schemas +# --------------------------------------------------------------------------- +PROTO_DIR="${EXTERNAL_DIR}/foxglove-sdk/schemas/proto" + +if [ ! -d "${PROTO_DIR}/foxglove" ]; then + echo "ERROR: Proto schemas not found at ${PROTO_DIR}/foxglove" + exit 1 +fi + +mkdir -p "${GENERATED_DIR}" + +echo "Generating protobuf C++ sources ..." +protoc \ + --cpp_out="${GENERATED_DIR}" \ + -I "${PROTO_DIR}" \ + "${PROTO_DIR}"/foxglove/*.proto + +echo "" +echo "Setup complete. Generated files are in:" +echo " ${GENERATED_DIR}" +echo "" +echo "To build:" +echo " cd ${PROJECT_DIR}" +echo " cmake -B build && cmake --build build" diff --git a/include/livekit/data_frame.h b/include/livekit/data_frame.h new file mode 100644 index 0000000..6bfa5e0 --- /dev/null +++ b/include/livekit/data_frame.h @@ -0,0 +1,45 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +namespace livekit { + +/** + * A single frame of data published or received on a data track. + * + * Carries an arbitrary binary payload and an optional user-specified + * timestamp. The unit is application-defined; the SDK examples use + * microseconds since the Unix epoch (system_clock). + */ +struct DataFrame { + /** Arbitrary binary payload (the frame contents). */ + std::vector payload; + + /** + * Optional application-defined timestamp. + * + * The proto field is a bare uint64 with no prescribed unit. + * By convention the SDK examples use microseconds since the Unix epoch. + */ + std::optional user_timestamp; +}; + +} // namespace livekit diff --git a/include/livekit/data_track_frame.h b/include/livekit/data_track_frame.h new file mode 100644 index 0000000..6bfa5e0 --- /dev/null +++ b/include/livekit/data_track_frame.h @@ -0,0 +1,45 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +namespace livekit { + +/** + * A single frame of data published or received on a data track. + * + * Carries an arbitrary binary payload and an optional user-specified + * timestamp. The unit is application-defined; the SDK examples use + * microseconds since the Unix epoch (system_clock). + */ +struct DataFrame { + /** Arbitrary binary payload (the frame contents). */ + std::vector payload; + + /** + * Optional application-defined timestamp. + * + * The proto field is a bare uint64 with no prescribed unit. + * By convention the SDK examples use microseconds since the Unix epoch. + */ + std::optional user_timestamp; +}; + +} // namespace livekit diff --git a/include/livekit/data_track_info.h b/include/livekit/data_track_info.h new file mode 100644 index 0000000..45c4fc5 --- /dev/null +++ b/include/livekit/data_track_info.h @@ -0,0 +1,40 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +namespace livekit { + +/** + * Metadata about a published data track. + * + * Unlike audio/video tracks, data tracks are not part of the Track class + * hierarchy. They carry their own lightweight info struct. + */ +struct DataTrackInfo { + /** Publisher-assigned track name (unique per publisher). */ + std::string name; + + /** SFU-assigned track identifier. */ + std::string sid; + + /** Whether frames on this track use end-to-end encryption. */ + bool uses_e2ee = false; +}; + +} // namespace livekit diff --git a/include/livekit/data_track_subscription.h b/include/livekit/data_track_subscription.h new file mode 100644 index 0000000..ca76cb7 --- /dev/null +++ b/include/livekit/data_track_subscription.h @@ -0,0 +1,124 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "livekit/data_frame.h" +#include "livekit/ffi_handle.h" + +#include +#include +#include +#include +#include + +namespace livekit { + +namespace proto { +class FfiEvent; +} + +/** + * An active subscription to a remote data track. + * + * Provides a blocking read() interface similar to AudioStream / VideoStream. + * Frames are delivered via FfiEvent callbacks and queued internally. + * + * Dropping (destroying) the subscription automatically unsubscribes from the + * remote track by releasing the underlying FFI handle. + * + * Typical usage: + * + * auto sub = remoteDataTrack->subscribe(); + * DataFrame frame; + * while (sub->read(frame)) { + * // process frame.payload + * } + */ +class DataTrackSubscription { +public: + struct Options { + /// Maximum buffered frames (count). 0 = unbounded. + /// When non-zero, behaves as a ring buffer (oldest dropped on overflow). + std::size_t capacity{0}; + }; + + virtual ~DataTrackSubscription(); + + DataTrackSubscription(const DataTrackSubscription &) = delete; + DataTrackSubscription &operator=(const DataTrackSubscription &) = delete; + DataTrackSubscription(DataTrackSubscription &&) noexcept; + DataTrackSubscription &operator=(DataTrackSubscription &&) noexcept; + + /** + * Blocking read: waits until a DataFrame is available, or the + * subscription reaches EOS / is closed. + * + * @param out On success, filled with the next data frame. + * @return true if a frame was delivered; false if the subscription ended. + */ + bool read(DataFrame &out); + + /** + * End the subscription early. + * + * Releases the FFI handle (which unsubscribes from the remote track), + * unregisters the event listener, and wakes any blocking read(). + */ + void close(); + +private: + friend class RemoteDataTrack; + + DataTrackSubscription() = default; + /// Internal init helper, called by RemoteDataTrack. + void init(FfiHandle subscription_handle, const Options &options); + + /// FFI event handler, called by FfiClient. + void onFfiEvent(const proto::FfiEvent &event); + + /// Push a received DataFrame to the internal queue. + void pushFrame(DataFrame &&frame); + + /// Push an end-of-stream signal (EOS). + void pushEos(); + + /** Protects all mutable state below. */ + mutable std::mutex mutex_; + + /** Signalled when a frame is pushed or the subscription ends. */ + std::condition_variable cv_; + + /** FIFO of received frames awaiting read(). */ + std::deque queue_; + + /** Max buffered frames (0 = unbounded). Oldest dropped on overflow. */ + std::size_t capacity_{0}; + + /** True once the remote side signals end-of-stream. */ + bool eof_{false}; + + /** True after close() has been called by the consumer. */ + bool closed_{false}; + + /** RAII handle for the Rust-owned subscription resource. */ + FfiHandle subscription_handle_; + + /** FfiClient listener id for routing FfiEvent callbacks to this object. */ + std::int64_t listener_id_{0}; +}; + +} // namespace livekit diff --git a/include/livekit/local_data_track.h b/include/livekit/local_data_track.h new file mode 100644 index 0000000..285e993 --- /dev/null +++ b/include/livekit/local_data_track.h @@ -0,0 +1,85 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "livekit/data_frame.h" +#include "livekit/data_track_info.h" +#include "livekit/ffi_handle.h" + +#include +#include + +namespace livekit { + +namespace proto { +class OwnedLocalDataTrack; +} + +/** + * Represents a locally published data track. + * + * Unlike audio/video tracks, data tracks do not extend the Track base class. + * They use a separate publish/unpublish lifecycle and carry arbitrary binary + * frames instead of media. + * + * Created via LocalParticipant::publishDataTrack(). + * + * Typical usage: + * + * auto lp = room->localParticipant(); + * auto dt = lp->publishDataTrack("sensor-data"); + * DataFrame frame; + * frame.payload = {0x01, 0x02, 0x03}; + * dt->tryPush(frame); + * lp->unpublishDataTrack(dt); + */ +class LocalDataTrack { +public: + ~LocalDataTrack() = default; + + LocalDataTrack(const LocalDataTrack &) = delete; + LocalDataTrack &operator=(const LocalDataTrack &) = delete; + + /// Metadata about this data track. + const DataTrackInfo &info() const noexcept { return info_; } + + /** + * Try to push a frame to all subscribers of this track. + * + * @return true on success, false if the push failed (e.g. back-pressure + * or the track has been unpublished). + */ + bool tryPush(const DataFrame &frame); + + /// Whether the track is still published in the room. + bool isPublished() const; + +private: + friend class LocalParticipant; + + explicit LocalDataTrack(const proto::OwnedLocalDataTrack &owned); + + uintptr_t ffi_handle_id() const noexcept { return handle_.get(); } + + /** RAII wrapper for the Rust-owned FFI resource. */ + FfiHandle handle_; + + /** Metadata snapshot taken at construction time. */ + DataTrackInfo info_; +}; + +} // namespace livekit diff --git a/include/livekit/local_participant.h b/include/livekit/local_participant.h index a624238..9391f0b 100644 --- a/include/livekit/local_participant.h +++ b/include/livekit/local_participant.h @@ -17,6 +17,7 @@ #pragma once #include "livekit/ffi_handle.h" +#include "livekit/local_data_track.h" #include "livekit/participant.h" #include "livekit/room_event_types.h" #include "livekit/rpc_error.h" @@ -95,7 +96,13 @@ class LocalParticipant : public Participant { const std::string &topic = {}); /** - * Publish SIP DTMF message. + * Publish a SIP DTMF (phone keypad) tone into the room. + * + * Only meaningful when a SIP trunk is bridging a phone call into the + * room. See SipDtmfData for background on SIP and DTMF. + * + * @param code DTMF code (0-15). + * @param digit Human-readable digit string (e.g. "5", "#"). */ void publishDtmf(int code, const std::string &digit); @@ -135,6 +142,30 @@ class LocalParticipant : public Participant { */ void unpublishTrack(const std::string &track_sid); + /** + * Publish a data track to the room. + * + * Data tracks carry arbitrary binary frames and are independent of the + * audio/video track hierarchy. The returned LocalDataTrack can push + * frames via tryPush() and be unpublished via unpublishDataTrack(). + * + * @param name Unique track name visible to other participants. + * @return Shared pointer to the published data track. + * @throws std::runtime_error on FFI or publish failure. + */ + std::shared_ptr + publishDataTrack(const std::string &name); + + /** + * Unpublish a data track from the room. + * + * After this call, tryPush() on the track will fail and the track + * cannot be re-published. + * + * @param track The data track to unpublish. Must not be null. + */ + void unpublishDataTrack(const std::shared_ptr &track); + /** * Initiate an RPC call to a remote participant. * diff --git a/include/livekit/remote_data_track.h b/include/livekit/remote_data_track.h new file mode 100644 index 0000000..a457902 --- /dev/null +++ b/include/livekit/remote_data_track.h @@ -0,0 +1,93 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "livekit/data_track_info.h" +#include "livekit/data_track_subscription.h" +#include "livekit/ffi_handle.h" + +#include +#include + +namespace livekit { + +namespace proto { +class OwnedRemoteDataTrack; +} + +/** + * Represents a data track published by a remote participant. + * + * Discovered via the RemoteDataTrackPublishedEvent room event. Unlike + * audio/video tracks, remote data tracks require an explicit subscribe() + * call to begin receiving frames. + * + * Typical usage: + * + * // In RoomDelegate::onRemoteDataTrackPublished callback: + * auto sub = remoteDataTrack->subscribe(); + * DataFrame frame; + * while (sub->read(frame)) { + * // process frame + * } + */ +class RemoteDataTrack { +public: + ~RemoteDataTrack() = default; + + RemoteDataTrack(const RemoteDataTrack &) = delete; + RemoteDataTrack &operator=(const RemoteDataTrack &) = delete; + + /// Metadata about this data track. + const DataTrackInfo &info() const noexcept { return info_; } + + /// Identity of the remote participant who published this track. + const std::string &publisherIdentity() const noexcept { + return publisher_identity_; + } + + /// Whether the track is still published by the remote participant. + bool isPublished() const; + + /** + * Subscribe to this remote data track. + * + * Returns a DataTrackSubscription that delivers frames via blocking + * read(). Destroy the subscription to unsubscribe. + * + * @throws std::runtime_error if the FFI subscribe call fails. + */ + std::shared_ptr + subscribe(const DataTrackSubscription::Options &options = {}); + +private: + friend class Room; + + explicit RemoteDataTrack(const proto::OwnedRemoteDataTrack &owned); + + uintptr_t ffi_handle_id() const noexcept { return handle_.get(); } + /** RAII wrapper for the Rust-owned FFI resource. */ + FfiHandle handle_; + + /** Metadata snapshot taken at construction time. */ + DataTrackInfo info_; + + /** Identity string of the remote participant who published this track. */ + std::string publisher_identity_; +}; + +} // namespace livekit diff --git a/include/livekit/room_delegate.h b/include/livekit/room_delegate.h index 04474a9..7f7598f 100644 --- a/include/livekit/room_delegate.h +++ b/include/livekit/room_delegate.h @@ -287,6 +287,19 @@ class RoomDelegate { */ virtual void onTextStreamOpened(Room &, const TextStreamOpenedEvent &) {} + // ------------------------------------------------------------------ + // Data tracks + // ------------------------------------------------------------------ + + /** + * Called when a remote participant publishes a data track. + * + * Data tracks are independent of the audio/video track hierarchy and + * require an explicit subscribe() call to start receiving frames. + */ + virtual void + onRemoteDataTrackPublished(Room &, const RemoteDataTrackPublishedEvent &) {} + // ------------------------------------------------------------------ // Participants snapshot // ------------------------------------------------------------------ diff --git a/include/livekit/room_event_types.h b/include/livekit/room_event_types.h index 63c7514..337a6ae 100644 --- a/include/livekit/room_event_types.h +++ b/include/livekit/room_event_types.h @@ -29,6 +29,7 @@ namespace livekit { class Track; class Participant; class RemoteParticipant; +class RemoteDataTrack; class LocalTrackPublication; class RemoteTrackPublication; class TrackPublication; @@ -100,7 +101,7 @@ enum class DisconnectReason { RoomClosed, UserUnavailable, UserRejected, - SipTrunkFailure, + SipTrunkFailure, ///< SIP (telephony) trunk connection failed ConnectionTimeout, MediaFailure }; @@ -117,10 +118,17 @@ struct UserPacketData { }; /** - * SIP DTMF payload carried via data packets. + * SIP (Session Initiation Protocol) DTMF payload carried via data packets. + * + * SIP is a signalling protocol used in VoIP telephony. LiveKit supports + * SIP trunking, which bridges traditional phone calls into LiveKit rooms. + * DTMF (Dual-Tone Multi-Frequency) tones are the signals generated when + * phone keypad buttons are pressed (0-9, *, #). This struct surfaces + * those tones so that applications handling SIP-bridged calls can react + * to caller input (e.g. IVR menu selection). */ struct SipDtmfData { - /** DTMF code value. */ + /** Numeric DTMF code (0-15, mapping to 0-9, *, #, A-D). */ std::uint32_t code = 0; /** Human-readable digit representation (e.g. "1", "#"). */ @@ -719,4 +727,16 @@ struct E2eeStateChangedEvent { EncryptionState state = EncryptionState::New; }; +/** + * Fired when a remote participant publishes a data track. + * + * Data tracks are independent of the audio/video track hierarchy. + * The application must call RemoteDataTrack::subscribe() to start + * receiving frames. + */ +struct RemoteDataTrackPublishedEvent { + /** The newly published remote data track. */ + std::shared_ptr track; +}; + } // namespace livekit diff --git a/include/livekit/track.h b/include/livekit/track.h index 2487d39..850e359 100644 --- a/include/livekit/track.h +++ b/include/livekit/track.h @@ -88,7 +88,7 @@ class Track { std::optional mime_type() const noexcept { return mime_type_; } // Handle access - bool has_handle() const noexcept { return !handle_.valid(); } + bool has_handle() const noexcept { return handle_.valid(); } uintptr_t ffi_handle_id() const noexcept { return handle_.get(); } // Async get stats diff --git a/src/data_track_subscription.cpp b/src/data_track_subscription.cpp new file mode 100644 index 0000000..94e34c5 --- /dev/null +++ b/src/data_track_subscription.cpp @@ -0,0 +1,171 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "livekit/data_track_subscription.h" + +#include "data_track.pb.h" +#include "ffi.pb.h" +#include "ffi_client.h" + +#include + +namespace livekit { + +using proto::FfiEvent; + +DataTrackSubscription::~DataTrackSubscription() { close(); } + +DataTrackSubscription::DataTrackSubscription( + DataTrackSubscription &&other) noexcept { + std::lock_guard lock(other.mutex_); + queue_ = std::move(other.queue_); + capacity_ = other.capacity_; + eof_ = other.eof_; + closed_ = other.closed_; + subscription_handle_ = std::move(other.subscription_handle_); + listener_id_ = other.listener_id_; + + other.listener_id_ = 0; + other.closed_ = true; +} + +DataTrackSubscription & +DataTrackSubscription::operator=(DataTrackSubscription &&other) noexcept { + if (this == &other) { + return *this; + } + + close(); + + { + std::lock_guard lock_this(mutex_); + std::lock_guard lock_other(other.mutex_); + + queue_ = std::move(other.queue_); + capacity_ = other.capacity_; + eof_ = other.eof_; + closed_ = other.closed_; + subscription_handle_ = std::move(other.subscription_handle_); + listener_id_ = other.listener_id_; + + other.listener_id_ = 0; + other.closed_ = true; + } + + return *this; +} + +void DataTrackSubscription::init(FfiHandle subscription_handle, + const Options &options) { + subscription_handle_ = std::move(subscription_handle); + capacity_ = options.capacity; + + listener_id_ = FfiClient::instance().AddListener( + [this](const FfiEvent &e) { this->onFfiEvent(e); }); +} + +bool DataTrackSubscription::read(DataFrame &out) { + std::unique_lock lock(mutex_); + + cv_.wait(lock, [this] { return !queue_.empty() || eof_ || closed_; }); + + if (closed_ || (queue_.empty() && eof_)) { + return false; + } + + out = std::move(queue_.front()); + queue_.pop_front(); + return true; +} + +void DataTrackSubscription::close() { + { + std::lock_guard lock(mutex_); + if (closed_) { + return; + } + closed_ = true; + } + + if (subscription_handle_.get() != 0) { + subscription_handle_.reset(); + } + + if (listener_id_ != 0) { + FfiClient::instance().RemoveListener(listener_id_); + listener_id_ = 0; + } + + cv_.notify_all(); +} + +void DataTrackSubscription::onFfiEvent(const FfiEvent &event) { + if (event.message_case() != FfiEvent::kDataTrackSubscriptionEvent) { + return; + } + + const auto &dts = event.data_track_subscription_event(); + if (dts.subscription_handle() != + static_cast(subscription_handle_.get())) { + return; + } + + if (dts.has_frame_received()) { + const auto &fr = dts.frame_received().frame(); + DataFrame frame; + const auto &payload_str = fr.payload(); + frame.payload.assign( + reinterpret_cast(payload_str.data()), + reinterpret_cast(payload_str.data()) + + payload_str.size()); + if (fr.has_user_timestamp()) { + frame.user_timestamp = fr.user_timestamp(); + } + pushFrame(std::move(frame)); + } else if (dts.has_eos()) { + pushEos(); + } +} + +void DataTrackSubscription::pushFrame(DataFrame &&frame) { + { + std::lock_guard lock(mutex_); + + if (closed_ || eof_) { + return; + } + + if (capacity_ > 0 && queue_.size() >= capacity_) { + queue_.pop_front(); + } + + queue_.push_back(std::move(frame)); + } + cv_.notify_one(); +} + +void DataTrackSubscription::pushEos() { + { + std::lock_guard lock(mutex_); + if (eof_) { + return; + } + eof_ = true; + } + cv_.notify_all(); +} + +} // namespace livekit diff --git a/src/ffi_client.cpp b/src/ffi_client.cpp index 2cc022b..60fca7f 100644 --- a/src/ffi_client.cpp +++ b/src/ffi_client.cpp @@ -16,6 +16,7 @@ #include +#include "data_track.pb.h" #include "e2ee.pb.h" #include "ffi.pb.h" #include "ffi_client.h" @@ -114,6 +115,12 @@ std::optional ExtractAsyncId(const proto::FfiEvent &event) { case E::kSendBytes: return event.send_bytes().async_id(); + // data track async completions + case E::kPublishDataTrack: + return event.publish_data_track().async_id(); + case E::kSubscribeDataTrack: + return event.subscribe_data_track().async_id(); + // NOT async completion: case E::kRoomEvent: case E::kTrackEvent: @@ -121,6 +128,7 @@ std::optional ExtractAsyncId(const proto::FfiEvent &event) { case E::kAudioStreamEvent: case E::kByteStreamReaderEvent: case E::kTextStreamReaderEvent: + case E::kDataTrackSubscriptionEvent: case E::kRpcMethodInvocation: case E::kLogs: case E::kPanic: @@ -316,7 +324,12 @@ FfiClient::connectAsync(const std::string &url, const std::string &token, auto *opts = connect->mutable_options(); opts->set_auto_subscribe(options.auto_subscribe); opts->set_dynacast(options.dynacast); - opts->set_single_peer_connection(options.single_peer_connection); + // opts->set_single_peer_connection(options.single_peer_connection); + LK_LOG_INFO("[FfiClient] connectAsync: auto_subscribe={}, dynacast={}, " + "single_peer_connection={}", + options.auto_subscribe, options.dynacast, + options.single_peer_connection); + // --- E2EE / encryption (optional) --- if (options.encryption.has_value()) { const E2EEOptions &e2ee = *options.encryption; @@ -607,6 +620,99 @@ std::future FfiClient::publishDataAsync( return fut; } +std::future +FfiClient::publishDataTrackAsync(std::uint64_t local_participant_handle, + const std::string &track_name) { + const AsyncId async_id = generateAsyncId(); + + auto fut = registerAsync( + async_id, + [async_id](const proto::FfiEvent &event) { + return event.has_publish_data_track() && + event.publish_data_track().async_id() == async_id; + }, + [](const proto::FfiEvent &event, + std::promise &pr) { + const auto &cb = event.publish_data_track(); + if (cb.has_error() && !cb.error().empty()) { + pr.set_exception( + std::make_exception_ptr(std::runtime_error(cb.error()))); + return; + } + if (!cb.has_track()) { + pr.set_exception(std::make_exception_ptr( + std::runtime_error("PublishDataTrackCallback missing track"))); + return; + } + proto::OwnedLocalDataTrack track = cb.track(); + pr.set_value(std::move(track)); + }); + + proto::FfiRequest req; + auto *msg = req.mutable_publish_data_track(); + msg->set_local_participant_handle(local_participant_handle); + msg->mutable_options()->set_name(track_name); + msg->set_request_async_id(async_id); + + try { + proto::FfiResponse resp = sendRequest(req); + if (!resp.has_publish_data_track()) { + logAndThrow("FfiResponse missing publish_data_track"); + } + } catch (...) { + cancelPendingByAsyncId(async_id); + throw; + } + + return fut; +} + +std::future +FfiClient::subscribeDataTrackAsync(std::uint64_t track_handle) { + const AsyncId async_id = generateAsyncId(); + + auto fut = registerAsync( + async_id, + [async_id](const proto::FfiEvent &event) { + return event.has_subscribe_data_track() && + event.subscribe_data_track().async_id() == async_id; + }, + [](const proto::FfiEvent &event, + std::promise &pr) { + const auto &cb = event.subscribe_data_track(); + if (cb.has_error() && !cb.error().empty()) { + pr.set_exception( + std::make_exception_ptr(std::runtime_error(cb.error()))); + return; + } + if (!cb.has_subscription()) { + pr.set_exception(std::make_exception_ptr(std::runtime_error( + "SubscribeDataTrackCallback missing subscription"))); + return; + } + proto::OwnedDataTrackSubscription sub = cb.subscription(); + pr.set_value(std::move(sub)); + }); + + proto::FfiRequest req; + auto *msg = req.mutable_subscribe_data_track(); + msg->set_track_handle(track_handle); + msg->mutable_options(); + msg->set_request_async_id(async_id); + + try { + proto::FfiResponse resp = sendRequest(req); + if (!resp.has_subscribe_data_track()) { + logAndThrow("FfiResponse missing subscribe_data_track"); + } + } catch (...) { + cancelPendingByAsyncId(async_id); + throw; + } + + return fut; +} + std::future FfiClient::publishSipDtmfAsync( std::uint64_t local_participant_handle, std::uint32_t code, const std::string &digit, diff --git a/src/ffi_client.h b/src/ffi_client.h index 667100e..327dbd9 100644 --- a/src/ffi_client.h +++ b/src/ffi_client.h @@ -38,6 +38,8 @@ class FfiEvent; class FfiResponse; class FfiRequest; class OwnedTrackPublication; +class OwnedLocalDataTrack; +class OwnedDataTrackSubscription; class DataStream; } // namespace proto @@ -123,6 +125,13 @@ class FfiClient { const std::string &payload, std::optional response_timeout_ms = std::nullopt); + // Data Track APIs + std::future + publishDataTrackAsync(std::uint64_t local_participant_handle, + const std::string &track_name); + std::future + subscribeDataTrackAsync(std::uint64_t track_handle); + // Data stream functionalities std::future sendStreamHeaderAsync(std::uint64_t local_participant_handle, diff --git a/src/local_data_track.cpp b/src/local_data_track.cpp new file mode 100644 index 0000000..1b290ae --- /dev/null +++ b/src/local_data_track.cpp @@ -0,0 +1,65 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "livekit/local_data_track.h" + +#include "data_track.pb.h" +#include "ffi.pb.h" +#include "ffi_client.h" + +namespace livekit { + +LocalDataTrack::LocalDataTrack(const proto::OwnedLocalDataTrack &owned) + : handle_(static_cast(owned.handle().id())) { + const auto &pi = owned.info(); + info_.name = pi.name(); + info_.sid = pi.sid(); + info_.uses_e2ee = pi.uses_e2ee(); +} + +bool LocalDataTrack::tryPush(const DataFrame &frame) { + if (!handle_.valid()) { + return false; + } + + proto::FfiRequest req; + auto *msg = req.mutable_local_data_track_try_push(); + msg->set_track_handle(static_cast(handle_.get())); + auto *pf = msg->mutable_frame(); + pf->set_payload(frame.payload.data(), frame.payload.size()); + if (frame.user_timestamp.has_value()) { + pf->set_user_timestamp(frame.user_timestamp.value()); + } + + proto::FfiResponse resp = FfiClient::instance().sendRequest(req); + const auto &r = resp.local_data_track_try_push(); + return !r.has_error(); +} + +bool LocalDataTrack::isPublished() const { + if (!handle_.valid()) { + return false; + } + + proto::FfiRequest req; + auto *msg = req.mutable_local_data_track_is_published(); + msg->set_track_handle(static_cast(handle_.get())); + + proto::FfiResponse resp = FfiClient::instance().sendRequest(req); + return resp.local_data_track_is_published().is_published(); +} + +} // namespace livekit diff --git a/src/local_participant.cpp b/src/local_participant.cpp index aa1b869..b0226af 100644 --- a/src/local_participant.cpp +++ b/src/local_participant.cpp @@ -17,10 +17,12 @@ #include "livekit/local_participant.h" #include "livekit/ffi_handle.h" +#include "livekit/local_data_track.h" #include "livekit/local_track_publication.h" #include "livekit/room_delegate.h" #include "livekit/track.h" +#include "data_track.pb.h" #include "ffi.pb.h" #include "ffi_client.h" #include "participant.pb.h" @@ -223,6 +225,39 @@ void LocalParticipant::unpublishTrack(const std::string &track_sid) { track_publications_.erase(track_sid); } +std::shared_ptr +LocalParticipant::publishDataTrack(const std::string &name) { + auto handle_id = ffiHandleId(); + if (handle_id == 0) { + throw std::runtime_error( + "LocalParticipant::publishDataTrack: invalid FFI handle"); + } + + auto fut = FfiClient::instance().publishDataTrackAsync( + static_cast(handle_id), name); + + proto::OwnedLocalDataTrack owned = fut.get(); + return std::shared_ptr(new LocalDataTrack(owned)); +} + +void LocalParticipant::unpublishDataTrack( + const std::shared_ptr &track) { + if (!track) { + return; + } + + auto handle_id = track->ffi_handle_id(); + if (handle_id == 0) { + return; + } + + proto::FfiRequest req; + auto *msg = req.mutable_local_data_track_unpublish(); + msg->set_track_handle(static_cast(handle_id)); + + (void)FfiClient::instance().sendRequest(req); +} + std::string LocalParticipant::performRpc( const std::string &destination_identity, const std::string &method, const std::string &payload, std::optional response_timeout) { diff --git a/src/remote_data_track.cpp b/src/remote_data_track.cpp new file mode 100644 index 0000000..6104b2b --- /dev/null +++ b/src/remote_data_track.cpp @@ -0,0 +1,68 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "livekit/remote_data_track.h" + +#include "data_track.pb.h" +#include "ffi.pb.h" +#include "ffi_client.h" + +#include + +namespace livekit { + +RemoteDataTrack::RemoteDataTrack(const proto::OwnedRemoteDataTrack &owned) + : handle_(static_cast(owned.handle().id())), + publisher_identity_(owned.publisher_identity()) { + const auto &pi = owned.info(); + info_.name = pi.name(); + info_.sid = pi.sid(); + info_.uses_e2ee = pi.uses_e2ee(); +} + +bool RemoteDataTrack::isPublished() const { + if (!handle_.valid()) { + return false; + } + + proto::FfiRequest req; + auto *msg = req.mutable_remote_data_track_is_published(); + msg->set_track_handle(static_cast(handle_.get())); + + proto::FfiResponse resp = FfiClient::instance().sendRequest(req); + return resp.remote_data_track_is_published().is_published(); +} + +std::shared_ptr +RemoteDataTrack::subscribe(const DataTrackSubscription::Options &options) { + if (!handle_.valid()) { + throw std::runtime_error("RemoteDataTrack::subscribe: invalid FFI handle"); + } + + auto fut = FfiClient::instance().subscribeDataTrackAsync( + static_cast(handle_.get())); + + proto::OwnedDataTrackSubscription owned_sub = fut.get(); + + FfiHandle sub_handle(static_cast(owned_sub.handle().id())); + + auto subscription = + std::shared_ptr(new DataTrackSubscription()); + subscription->init(std::move(sub_handle), options); + return subscription; +} + +} // namespace livekit diff --git a/src/room.cpp b/src/room.cpp index 7eed7ed..428c94a 100644 --- a/src/room.cpp +++ b/src/room.cpp @@ -21,6 +21,7 @@ #include "livekit/local_participant.h" #include "livekit/local_track_publication.h" #include "livekit/remote_audio_track.h" +#include "livekit/remote_data_track.h" #include "livekit/remote_participant.h" #include "livekit/remote_track_publication.h" #include "livekit/remote_video_track.h" @@ -28,6 +29,7 @@ #include "livekit/room_event_types.h" #include "livekit/video_stream.h" +#include "data_track.pb.h" #include "ffi.pb.h" #include "ffi_client.h" #include "livekit_ffi.h" @@ -598,6 +600,24 @@ void Room::OnEvent(const FfiEvent &event) { } break; } + case proto::RoomEvent::kRemoteDataTrackPublished: { + const auto &rdtp = re.remote_data_track_published(); + auto remote_track = + std::shared_ptr(new RemoteDataTrack(rdtp.track())); + LK_LOG_INFO("[Room] RoomEvent::kRemoteDataTrackPublished: \"{}\" from " + "\"{}\" (sid={})", + remote_track->info().name, remote_track->publisherIdentity(), + remote_track->info().sid); + RemoteDataTrackPublishedEvent ev; + ev.track = remote_track; + if (delegate_snapshot) { + delegate_snapshot->onRemoteDataTrackPublished(*this, ev); + } else { + LK_LOG_ERROR("[Room] No delegate set; RemoteDataTrackPublished " + "event dropped."); + } + break; + } case proto::RoomEvent::kTrackMuted: { TrackMutedEvent ev; bool success = false;
LiveKit Ecosystem