From b5f0334674d0e25e6656266471a1225ca519ee1c Mon Sep 17 00:00:00 2001 From: jprochazk Date: Sun, 26 Jan 2025 22:16:30 +0100 Subject: [PATCH 01/87] Make `re_grpc_client` a non-optional dependency Instead, move dataplatform-related stuff into `redap` and feature gate that behind a separate feature flag. Message proxy client is now always available. --- crates/store/re_data_source/Cargo.toml | 4 +- .../store/re_data_source/src/data_source.rs | 6 +- crates/store/re_grpc_client/Cargo.toml | 5 + crates/store/re_grpc_client/src/lib.rs | 582 +----------------- .../mod.rs} | 0 .../re_grpc_client/src/{ => redap}/address.rs | 0 crates/store/re_grpc_client/src/redap/mod.rs | 576 +++++++++++++++++ crates/top/re_sdk/Cargo.toml | 4 +- crates/viewer/re_viewer/Cargo.toml | 4 +- crates/viewer/re_viewer/src/web_tools.rs | 13 +- rerun_py/src/remote.rs | 2 +- 11 files changed, 596 insertions(+), 600 deletions(-) rename crates/store/re_grpc_client/src/{message_proxy.rs => message_proxy/mod.rs} (100%) rename crates/store/re_grpc_client/src/{ => redap}/address.rs (100%) create mode 100644 crates/store/re_grpc_client/src/redap/mod.rs diff --git a/crates/store/re_data_source/Cargo.toml b/crates/store/re_data_source/Cargo.toml index df33d6d38bab..b47ea36006a1 100644 --- a/crates/store/re_data_source/Cargo.toml +++ b/crates/store/re_data_source/Cargo.toml @@ -23,11 +23,12 @@ all-features = true default = ["grpc"] ## Enable the gRPC Rerun Data Platform data source. -grpc = ["dep:re_grpc_client"] +grpc = ["re_grpc_client/redap"] [dependencies] re_data_loader.workspace = true +re_grpc_client.workspace = true re_log_encoding = { workspace = true, features = [ "decoder", "stream_from_http", @@ -43,7 +44,6 @@ itertools.workspace = true rayon.workspace = true # Optional dependencies: -re_grpc_client = { workspace = true, optional = true } [build-dependencies] re_build_tools.workspace = true diff --git a/crates/store/re_data_source/src/data_source.rs b/crates/store/re_data_source/src/data_source.rs index c2c7c7f11941..f3a14b0a57eb 100644 --- a/crates/store/re_data_source/src/data_source.rs +++ b/crates/store/re_data_source/src/data_source.rs @@ -37,7 +37,6 @@ pub enum DataSource { RerunGrpcUrl { url: String }, /// A stream of messages over gRPC, relayed from the SDK. - #[cfg(feature = "grpc")] MessageProxy { url: String }, } @@ -100,7 +99,6 @@ impl DataSource { } // TODO(#8761): URL prefix - #[cfg(feature = "grpc")] if uri.starts_with("temp://") { return Self::MessageProxy { url: uri }; } @@ -148,7 +146,6 @@ impl DataSource { Self::Stdin => None, #[cfg(feature = "grpc")] Self::RerunGrpcUrl { .. } => None, // TODO(jleibs): This needs to come from the server. - #[cfg(feature = "grpc")] Self::MessageProxy { .. } => None, } } @@ -259,10 +256,9 @@ impl DataSource { #[cfg(feature = "grpc")] Self::RerunGrpcUrl { url } => { - re_grpc_client::stream_from_redap(url, on_msg).map_err(|err| err.into()) + re_grpc_client::redap::stream_from_redap(url, on_msg).map_err(|err| err.into()) } - #[cfg(feature = "grpc")] Self::MessageProxy { url } => { re_grpc_client::message_proxy::stream(url, on_msg).map_err(|err| err.into()) } diff --git a/crates/store/re_grpc_client/Cargo.toml b/crates/store/re_grpc_client/Cargo.toml index 56a65f476ad4..d6e3dc12ae66 100644 --- a/crates/store/re_grpc_client/Cargo.toml +++ b/crates/store/re_grpc_client/Cargo.toml @@ -19,6 +19,11 @@ workspace = true all-features = true +[features] +default = ["redap"] + +redap = [] + [dependencies] re_arrow_util.workspace = true re_chunk.workspace = true diff --git a/crates/store/re_grpc_client/src/lib.rs b/crates/store/re_grpc_client/src/lib.rs index 9c063d094497..ce6eaf738c33 100644 --- a/crates/store/re_grpc_client/src/lib.rs +++ b/crates/store/re_grpc_client/src/lib.rs @@ -2,50 +2,8 @@ pub mod message_proxy; -use std::{collections::HashMap, error::Error, sync::Arc}; - -use arrow::{ - array::{ - Array as ArrowArray, ArrayRef as ArrowArrayRef, RecordBatch as ArrowRecordBatch, - StringArray as ArrowStringArray, - }, - datatypes::{DataType as ArrowDataType, Field as ArrowField}, -}; -use url::Url; - -use re_arrow_util::ArrowArrayDowncastRef as _; -use re_chunk::{Chunk, ChunkBuilder, ChunkId, EntityPath, RowId, Timeline, TransportChunk}; -use re_log_encoding::codec::{wire::decoder::Decode, CodecError}; -use re_log_types::{ - external::re_types_core::ComponentDescriptor, ApplicationId, BlueprintActivationCommand, - EntityPathFilter, LogMsg, SetStoreInfo, StoreId, StoreInfo, StoreKind, StoreSource, Time, -}; -use re_protos::{ - common::v0::RecordingId, - remote_store::v0::{ - storage_node_client::StorageNodeClient, CatalogFilter, FetchRecordingRequest, - QueryCatalogRequest, CATALOG_APP_ID_FIELD_NAME, CATALOG_ID_FIELD_NAME, - CATALOG_START_TIME_FIELD_NAME, - }, -}; -use re_types::{ - arrow_helpers::as_array_ref, - blueprint::{ - archetypes::{ContainerBlueprint, ViewBlueprint, ViewContents, ViewportBlueprint}, - components::{ContainerKind, RootContainer}, - }, - components::RecordingUri, - external::uuid, - Archetype, Component, -}; - -// ---------------------------------------------------------------------------- - -mod address; - -pub use address::{InvalidRedapAddress, RedapAddress}; - -// ---------------------------------------------------------------------------- +#[cfg(feature = "redap")] +pub mod redap; /// Wrapper with a nicer error message #[derive(Debug)] @@ -69,8 +27,8 @@ impl std::fmt::Display for TonicStatusError { } } -impl Error for TonicStatusError { - fn source(&self) -> Option<&(dyn Error + 'static)> { +impl std::error::Error for TonicStatusError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { self.0.source() } } @@ -86,7 +44,7 @@ pub enum StreamError { TonicStatus(#[from] TonicStatusError), #[error(transparent)] - CodecError(#[from] CodecError), + CodecError(#[from] re_log_encoding::codec::CodecError), #[error(transparent)] ChunkError(#[from] re_chunk::ChunkError), @@ -98,57 +56,6 @@ pub enum StreamError { InvalidUri(String), } -// ---------------------------------------------------------------------------- - -const CATALOG_BP_STORE_ID: &str = "catalog_blueprint"; -const CATALOG_REC_STORE_ID: &str = "catalog"; -const CATALOG_APPLICATION_ID: &str = "redap_catalog"; - -/// Stream an rrd file or metadata catalog over gRPC from a Rerun Data Platform server. -/// -/// `on_msg` can be used to wake up the UI thread on Wasm. -pub fn stream_from_redap( - url: String, - on_msg: Option>, -) -> Result, InvalidRedapAddress> { - re_log::debug!("Loading {url}…"); - - let address = url.as_str().try_into()?; - - let (tx, rx) = re_smart_channel::smart_channel( - re_smart_channel::SmartMessageSource::RerunGrpcStream { url: url.clone() }, - re_smart_channel::SmartChannelSource::RerunGrpcStream { url: url.clone() }, - ); - - spawn_future(async move { - match address { - RedapAddress::Recording { - redap_endpoint, - recording_id, - } => { - if let Err(err) = - stream_recording_async(tx, redap_endpoint, recording_id, on_msg).await - { - re_log::warn!( - "Error while streaming {url}: {}", - re_error::format_ref(&err) - ); - } - } - RedapAddress::Catalog { redap_endpoint } => { - if let Err(err) = stream_catalog_async(tx, redap_endpoint, on_msg).await { - re_log::warn!( - "Error while streaming {url}: {}", - re_error::format_ref(&err) - ); - } - } - } - }); - - Ok(rx) -} - #[cfg(target_arch = "wasm32")] fn spawn_future(future: F) where @@ -164,482 +71,3 @@ where { tokio::spawn(future); } - -async fn stream_recording_async( - tx: re_smart_channel::Sender, - redap_endpoint: Url, - recording_id: String, - on_msg: Option>, -) -> Result<(), StreamError> { - use tokio_stream::StreamExt as _; - - re_log::debug!("Connecting to {redap_endpoint}…"); - let mut client = { - #[cfg(target_arch = "wasm32")] - let tonic_client = tonic_web_wasm_client::Client::new_with_options( - redap_endpoint.to_string(), - tonic_web_wasm_client::options::FetchOptions::new(), - ); - - #[cfg(not(target_arch = "wasm32"))] - let tonic_client = tonic::transport::Endpoint::new(redap_endpoint.to_string())? - .connect() - .await?; - - // TODO(#8411): figure out the right size for this - StorageNodeClient::new(tonic_client).max_decoding_message_size(usize::MAX) - }; - - re_log::debug!("Fetching catalog data for {recording_id}…"); - - let resp = client - .query_catalog(QueryCatalogRequest { - column_projection: None, // fetch all columns - filter: Some(CatalogFilter { - recording_ids: vec![RecordingId { - id: recording_id.clone(), - }], - }), - }) - .await - .map_err(TonicStatusError)? - .into_inner() - .map(|resp| { - resp.and_then(|r| { - r.decode() - .map_err(|err| tonic::Status::internal(err.to_string())) - }) - }) - .collect::, tonic::Status>>() - .await - .map_err(TonicStatusError)?; - - if resp.len() != 1 || resp[0].num_rows() != 1 { - return Err(StreamError::ChunkError(re_chunk::ChunkError::Malformed { - reason: format!( - "expected exactly one recording with id {recording_id}, got {}", - resp.len() - ), - })); - } - - let store_info = - store_info_from_catalog_chunk(&TransportChunk::from(resp[0].clone()), &recording_id)?; - let store_id = store_info.store_id.clone(); - - re_log::debug!("Fetching {recording_id}…"); - - let mut resp = client - .fetch_recording(FetchRecordingRequest { - recording_id: Some(RecordingId { - id: recording_id.clone(), - }), - }) - .await - .map_err(TonicStatusError)? - .into_inner() - .map(|resp| { - resp.and_then(|r| { - r.decode() - .map_err(|err| tonic::Status::internal(err.to_string())) - }) - }); - - drop(client); - - // We need a whole StoreInfo here. - if tx - .send(LogMsg::SetStoreInfo(SetStoreInfo { - row_id: *re_chunk::RowId::new(), - info: store_info, - })) - .is_err() - { - re_log::debug!("Receiver disconnected"); - return Ok(()); - } - - re_log::info!("Starting to read..."); - while let Some(result) = resp.next().await { - let batch = result.map_err(TonicStatusError)?; - let chunk = Chunk::from_record_batch(batch)?; - - if tx - .send(LogMsg::ArrowMsg(store_id.clone(), chunk.to_arrow_msg()?)) - .is_err() - { - re_log::debug!("Receiver disconnected"); - return Ok(()); - } - - if let Some(on_msg) = &on_msg { - on_msg(); - } - } - - Ok(()) -} - -pub fn store_info_from_catalog_chunk( - tc: &TransportChunk, - recording_id: &str, -) -> Result { - let store_id = StoreId::from_string(StoreKind::Recording, recording_id.to_owned()); - - let (_field, data) = tc - .components() - .find(|(f, _)| f.name() == CATALOG_APP_ID_FIELD_NAME) - .ok_or(StreamError::ChunkError(re_chunk::ChunkError::Malformed { - reason: "no {CATALOG_APP_ID_FIELD_NAME} field found".to_owned(), - }))?; - let app_id = data - .downcast_array_ref::() - .ok_or(StreamError::ChunkError(re_chunk::ChunkError::Malformed { - reason: format!( - "{CATALOG_APP_ID_FIELD_NAME} must be a utf8 array: {:?}", - tc.schema_ref() - ), - }))? - .value(0); - - let (_field, data) = tc - .components() - .find(|(f, _)| f.name() == CATALOG_START_TIME_FIELD_NAME) - .ok_or(StreamError::ChunkError(re_chunk::ChunkError::Malformed { - reason: "no {CATALOG_START_TIME_FIELD_NAME}} field found".to_owned(), - }))?; - let start_time = data - .downcast_array_ref::() - .ok_or(StreamError::ChunkError(re_chunk::ChunkError::Malformed { - reason: format!( - "{CATALOG_START_TIME_FIELD_NAME} must be a Timestamp array: {:?}", - tc.schema_ref() - ), - }))? - .value(0); - - Ok(StoreInfo { - application_id: ApplicationId::from(app_id), - store_id: store_id.clone(), - cloned_from: None, - is_official_example: false, - started: Time::from_ns_since_epoch(start_time), - store_source: StoreSource::Unknown, - store_version: None, - }) -} - -async fn stream_catalog_async( - tx: re_smart_channel::Sender, - redap_endpoint: Url, - on_msg: Option>, -) -> Result<(), StreamError> { - use tokio_stream::StreamExt as _; - - re_log::debug!("Connecting to {redap_endpoint}…"); - let mut client = { - #[cfg(target_arch = "wasm32")] - let tonic_client = tonic_web_wasm_client::Client::new_with_options( - redap_endpoint.to_string(), - tonic_web_wasm_client::options::FetchOptions::new(), - ); - - #[cfg(not(target_arch = "wasm32"))] - let tonic_client = tonic::transport::Endpoint::new(redap_endpoint.to_string())? - .connect() - .await?; - - StorageNodeClient::new(tonic_client) - }; - - re_log::debug!("Fetching catalog…"); - - let mut resp = client - .query_catalog(QueryCatalogRequest { - column_projection: None, // fetch all columns - filter: None, // fetch all rows - }) - .await - .map_err(TonicStatusError)? - .into_inner() - .map(|resp| { - resp.and_then(|r| { - r.decode() - .map_err(|err| tonic::Status::internal(err.to_string())) - }) - }); - - drop(client); - - if activate_catalog_blueprint(&tx).is_err() { - re_log::debug!("Failed to activate catalog blueprint"); - return Ok(()); - } - - // Craft the StoreInfo for the actual catalog data - let store_id = StoreId::from_string(StoreKind::Recording, CATALOG_REC_STORE_ID.to_owned()); - - let store_info = StoreInfo { - application_id: ApplicationId::from(CATALOG_APPLICATION_ID), - store_id: store_id.clone(), - cloned_from: None, - is_official_example: false, - started: Time::now(), - store_source: StoreSource::Unknown, - store_version: None, - }; - - if tx - .send(LogMsg::SetStoreInfo(SetStoreInfo { - row_id: *re_chunk::RowId::new(), - info: store_info, - })) - .is_err() - { - re_log::debug!("Receiver disconnected"); - return Ok(()); - } - - re_log::info!("Starting to read..."); - while let Some(result) = resp.next().await { - let input = TransportChunk::from(result.map_err(TonicStatusError)?); - - // Catalog received from the ReDap server isn't suitable for direct conversion to a Rerun Chunk: - // - conversion expects "data" columns to be ListArrays, hence we need to convert any individual row column data to ListArray - // - conversion expects the input TransportChunk to have a ChunkId so we need to add that piece of metadata - - let mut fields: Vec = Vec::new(); - let mut columns: Vec = Vec::new(); - // add the (row id) control field - let (row_id_field, row_id_data) = input.controls().next().ok_or( - StreamError::ChunkError(re_chunk::ChunkError::Malformed { - reason: "no control field found".to_owned(), - }), - )?; - - fields.push( - ArrowField::new( - RowId::name().to_string(), // need to rename to Rerun Chunk expected control field - row_id_field.data_type().clone(), - false, /* not nullable */ - ) - .with_metadata(TransportChunk::field_metadata_control_column()), - ); - columns.push(row_id_data.clone()); - - // next add any timeline field - for (field, data) in input.timelines() { - fields.push(field.clone()); - columns.push(data.clone()); - } - - // now add all the 'data' fields - we slice each column array into individual arrays and then convert the whole lot into a ListArray - for (field, data) in input.components() { - let data_field_inner = - ArrowField::new("item", field.data_type().clone(), true /* nullable */); - - let data_field = ArrowField::new( - field.name().clone(), - ArrowDataType::List(Arc::new(data_field_inner.clone())), - false, /* not nullable */ - ) - .with_metadata(TransportChunk::field_metadata_data_column()); - - let mut sliced: Vec = Vec::new(); - for idx in 0..data.len() { - sliced.push(data.clone().slice(idx, 1)); - } - - let data_arrays = sliced.iter().map(|e| Some(e.as_ref())).collect::>(); - #[allow(clippy::unwrap_used)] // we know we've given the right field type - let data_field_array: arrow::array::ListArray = - re_arrow_util::arrow_util::arrays_to_list_array( - data_field_inner.data_type().clone(), - &data_arrays, - ) - .unwrap(); - - fields.push(data_field); - columns.push(as_array_ref(data_field_array)); - } - - let schema = { - let metadata = HashMap::from_iter([ - ( - TransportChunk::CHUNK_METADATA_KEY_ENTITY_PATH.to_owned(), - "catalog".to_owned(), - ), - ( - TransportChunk::CHUNK_METADATA_KEY_ID.to_owned(), - ChunkId::new().to_string(), - ), - ]); - arrow::datatypes::Schema::new_with_metadata(fields, metadata) - }; - - let record_batch = ArrowRecordBatch::try_new(schema.into(), columns) - .map_err(re_chunk::ChunkError::from)?; - let mut chunk = Chunk::from_record_batch(record_batch)?; - - // finally, enrich catalog data with RecordingUri that's based on the ReDap endpoint (that we know) - // and the recording id (that we have in the catalog data) - let host = redap_endpoint - .host() - .ok_or(StreamError::InvalidUri(format!( - "couldn't get host from {redap_endpoint}" - )))?; - let port = redap_endpoint - .port() - .ok_or(StreamError::InvalidUri(format!( - "couldn't get port from {redap_endpoint}" - )))?; - - let recording_uri_arrays: Vec = chunk - .iter_slices::(CATALOG_ID_FIELD_NAME.into()) - .map(|id| { - let rec_id = &id[0]; // each component batch is of length 1 i.e. single 'id' value - - let recording_uri = format!("rerun://{host}:{port}/recording/{rec_id}"); - - as_array_ref(ArrowStringArray::from(vec![recording_uri])) - }) - .collect(); - - let recording_id_arrays = recording_uri_arrays - .iter() - .map(|e| Some(e.as_ref())) - .collect::>(); - - let rec_id_field = ArrowField::new("item", ArrowDataType::Utf8, true); - #[allow(clippy::unwrap_used)] // we know we've given the right field type - let uris = re_arrow_util::arrow_util::arrays_to_list_array( - rec_id_field.data_type().clone(), - &recording_id_arrays, - ) - .unwrap(); - - chunk.add_component(ComponentDescriptor::new(RecordingUri::name()), uris)?; - - if tx - .send(LogMsg::ArrowMsg(store_id.clone(), chunk.to_arrow_msg()?)) - .is_err() - { - re_log::debug!("Receiver disconnected"); - return Ok(()); - } - - if let Some(on_msg) = &on_msg { - on_msg(); - } - } - - Ok(()) -} - -// Craft a blueprint from relevant chunks and activate it -// TODO(zehiko) - manual crafting of the blueprint as we have below will go away and be replaced -// by either a blueprint crafted using rust Blueprint API or a blueprint fetched from ReDap (#8470) -fn activate_catalog_blueprint( - tx: &re_smart_channel::Sender, -) -> Result<(), Box> { - let blueprint_store_id = - StoreId::from_string(StoreKind::Blueprint, CATALOG_BP_STORE_ID.to_owned()); - let blueprint_store_info = StoreInfo { - application_id: ApplicationId::from(CATALOG_APPLICATION_ID), - store_id: blueprint_store_id.clone(), - cloned_from: None, - is_official_example: false, - started: Time::now(), - store_source: StoreSource::Unknown, - store_version: None, - }; - - if tx - .send(LogMsg::SetStoreInfo(SetStoreInfo { - row_id: *re_chunk::RowId::new(), - info: blueprint_store_info, - })) - .is_err() - { - re_log::debug!("Receiver disconnected"); - return Ok(()); - } - - let timepoint = [(Timeline::new_sequence("blueprint"), 1)]; - - let vb = ViewBlueprint::new("Dataframe") - .with_visible(true) - .with_space_origin("/"); - - // TODO(zehiko) we shouldn't really be creating all these ids and entity paths manually... (#8470) - let view_uuid = uuid::Uuid::new_v4(); - let view_entity_path = format!("/view/{view_uuid}"); - let view_chunk = ChunkBuilder::new(ChunkId::new(), view_entity_path.clone().into()) - .with_archetype(RowId::new(), timepoint, &vb) - .build()?; - - let epf = EntityPathFilter::parse_forgiving("/**"); - let vc = ViewContents::new(epf.iter_expressions()); - let view_contents_chunk = ChunkBuilder::new( - ChunkId::new(), - format!( - "{}/{}", - view_entity_path.clone(), - ViewContents::name().short_name() - ) - .into(), - ) - .with_archetype(RowId::new(), timepoint, &vc) - .build()?; - - let rc = ContainerBlueprint::new(ContainerKind::Grid) - .with_contents(&[EntityPath::from(view_entity_path)]) - .with_visible(true); - - let container_uuid = uuid::Uuid::new_v4(); - let container_chunk = ChunkBuilder::new( - ChunkId::new(), - format!("/container/{container_uuid}").into(), - ) - .with_archetype(RowId::new(), timepoint, &rc) - .build()?; - - let vp = ViewportBlueprint::new().with_root_container(RootContainer(container_uuid.into())); - let viewport_chunk = ChunkBuilder::new(ChunkId::new(), "/viewport".into()) - .with_archetype(RowId::new(), timepoint, &vp) - .build()?; - - for chunk in &[ - view_chunk, - view_contents_chunk, - container_chunk, - viewport_chunk, - ] { - if tx - .send(LogMsg::ArrowMsg( - blueprint_store_id.clone(), - chunk.to_arrow_msg()?, - )) - .is_err() - { - re_log::debug!("Receiver disconnected"); - return Ok(()); - } - } - - let blueprint_activation = BlueprintActivationCommand { - blueprint_id: blueprint_store_id.clone(), - make_active: true, - make_default: true, - }; - - if tx - .send(LogMsg::BlueprintActivationCommand(blueprint_activation)) - .is_err() - { - re_log::debug!("Receiver disconnected"); - return Ok(()); - } - - Ok(()) -} diff --git a/crates/store/re_grpc_client/src/message_proxy.rs b/crates/store/re_grpc_client/src/message_proxy/mod.rs similarity index 100% rename from crates/store/re_grpc_client/src/message_proxy.rs rename to crates/store/re_grpc_client/src/message_proxy/mod.rs diff --git a/crates/store/re_grpc_client/src/address.rs b/crates/store/re_grpc_client/src/redap/address.rs similarity index 100% rename from crates/store/re_grpc_client/src/address.rs rename to crates/store/re_grpc_client/src/redap/address.rs diff --git a/crates/store/re_grpc_client/src/redap/mod.rs b/crates/store/re_grpc_client/src/redap/mod.rs new file mode 100644 index 000000000000..39630c0585a3 --- /dev/null +++ b/crates/store/re_grpc_client/src/redap/mod.rs @@ -0,0 +1,576 @@ +use std::{collections::HashMap, sync::Arc}; + +use arrow::{ + array::{ + Array as ArrowArray, ArrayRef as ArrowArrayRef, RecordBatch as ArrowRecordBatch, + StringArray as ArrowStringArray, + }, + datatypes::{DataType as ArrowDataType, Field as ArrowField}, +}; +use url::Url; + +use re_arrow_util::ArrowArrayDowncastRef as _; +use re_chunk::{Chunk, ChunkBuilder, ChunkId, EntityPath, RowId, Timeline, TransportChunk}; +use re_log_encoding::codec::wire::decoder::Decode; +use re_log_types::{ + external::re_types_core::ComponentDescriptor, ApplicationId, BlueprintActivationCommand, + EntityPathFilter, LogMsg, SetStoreInfo, StoreId, StoreInfo, StoreKind, StoreSource, Time, +}; +use re_protos::{ + common::v0::RecordingId, + remote_store::v0::{ + storage_node_client::StorageNodeClient, CatalogFilter, FetchRecordingRequest, + QueryCatalogRequest, CATALOG_APP_ID_FIELD_NAME, CATALOG_ID_FIELD_NAME, + CATALOG_START_TIME_FIELD_NAME, + }, +}; +use re_types::{ + arrow_helpers::as_array_ref, + blueprint::{ + archetypes::{ContainerBlueprint, ViewBlueprint, ViewContents, ViewportBlueprint}, + components::{ContainerKind, RootContainer}, + }, + components::RecordingUri, + external::uuid, + Archetype, Component, +}; + +// ---------------------------------------------------------------------------- + +mod address; + +pub use address::{InvalidRedapAddress, RedapAddress}; + +use crate::spawn_future; +use crate::StreamError; +use crate::TonicStatusError; + +// ---------------------------------------------------------------------------- + +const CATALOG_BP_STORE_ID: &str = "catalog_blueprint"; +const CATALOG_REC_STORE_ID: &str = "catalog"; +const CATALOG_APPLICATION_ID: &str = "redap_catalog"; + +/// Stream an rrd file or metadata catalog over gRPC from a Rerun Data Platform server. +/// +/// `on_msg` can be used to wake up the UI thread on Wasm. +pub fn stream_from_redap( + url: String, + on_msg: Option>, +) -> Result, InvalidRedapAddress> { + re_log::debug!("Loading {url}…"); + + let address = url.as_str().try_into()?; + + let (tx, rx) = re_smart_channel::smart_channel( + re_smart_channel::SmartMessageSource::RerunGrpcStream { url: url.clone() }, + re_smart_channel::SmartChannelSource::RerunGrpcStream { url: url.clone() }, + ); + + spawn_future(async move { + match address { + RedapAddress::Recording { + redap_endpoint, + recording_id, + } => { + if let Err(err) = + stream_recording_async(tx, redap_endpoint, recording_id, on_msg).await + { + re_log::warn!( + "Error while streaming {url}: {}", + re_error::format_ref(&err) + ); + } + } + RedapAddress::Catalog { redap_endpoint } => { + if let Err(err) = stream_catalog_async(tx, redap_endpoint, on_msg).await { + re_log::warn!( + "Error while streaming {url}: {}", + re_error::format_ref(&err) + ); + } + } + } + }); + + Ok(rx) +} + +async fn stream_recording_async( + tx: re_smart_channel::Sender, + redap_endpoint: Url, + recording_id: String, + on_msg: Option>, +) -> Result<(), StreamError> { + use tokio_stream::StreamExt as _; + + re_log::debug!("Connecting to {redap_endpoint}…"); + let mut client = { + #[cfg(target_arch = "wasm32")] + let tonic_client = tonic_web_wasm_client::Client::new_with_options( + redap_endpoint.to_string(), + tonic_web_wasm_client::options::FetchOptions::new(), + ); + + #[cfg(not(target_arch = "wasm32"))] + let tonic_client = tonic::transport::Endpoint::new(redap_endpoint.to_string())? + .connect() + .await?; + + // TODO(#8411): figure out the right size for this + StorageNodeClient::new(tonic_client).max_decoding_message_size(usize::MAX) + }; + + re_log::debug!("Fetching catalog data for {recording_id}…"); + + let resp = client + .query_catalog(QueryCatalogRequest { + column_projection: None, // fetch all columns + filter: Some(CatalogFilter { + recording_ids: vec![RecordingId { + id: recording_id.clone(), + }], + }), + }) + .await + .map_err(TonicStatusError)? + .into_inner() + .map(|resp| { + resp.and_then(|r| { + r.decode() + .map_err(|err| tonic::Status::internal(err.to_string())) + }) + }) + .collect::, tonic::Status>>() + .await + .map_err(TonicStatusError)?; + + if resp.len() != 1 || resp[0].num_rows() != 1 { + return Err(StreamError::ChunkError(re_chunk::ChunkError::Malformed { + reason: format!( + "expected exactly one recording with id {recording_id}, got {}", + resp.len() + ), + })); + } + + let store_info = + store_info_from_catalog_chunk(&TransportChunk::from(resp[0].clone()), &recording_id)?; + let store_id = store_info.store_id.clone(); + + re_log::debug!("Fetching {recording_id}…"); + + let mut resp = client + .fetch_recording(FetchRecordingRequest { + recording_id: Some(RecordingId { + id: recording_id.clone(), + }), + }) + .await + .map_err(TonicStatusError)? + .into_inner() + .map(|resp| { + resp.and_then(|r| { + r.decode() + .map_err(|err| tonic::Status::internal(err.to_string())) + }) + }); + + drop(client); + + // We need a whole StoreInfo here. + if tx + .send(LogMsg::SetStoreInfo(SetStoreInfo { + row_id: *re_chunk::RowId::new(), + info: store_info, + })) + .is_err() + { + re_log::debug!("Receiver disconnected"); + return Ok(()); + } + + re_log::info!("Starting to read..."); + while let Some(result) = resp.next().await { + let batch = result.map_err(TonicStatusError)?; + let chunk = Chunk::from_record_batch(batch)?; + + if tx + .send(LogMsg::ArrowMsg(store_id.clone(), chunk.to_arrow_msg()?)) + .is_err() + { + re_log::debug!("Receiver disconnected"); + return Ok(()); + } + + if let Some(on_msg) = &on_msg { + on_msg(); + } + } + + Ok(()) +} + +pub fn store_info_from_catalog_chunk( + tc: &TransportChunk, + recording_id: &str, +) -> Result { + let store_id = StoreId::from_string(StoreKind::Recording, recording_id.to_owned()); + + let (_field, data) = tc + .components() + .find(|(f, _)| f.name() == CATALOG_APP_ID_FIELD_NAME) + .ok_or(StreamError::ChunkError(re_chunk::ChunkError::Malformed { + reason: "no {CATALOG_APP_ID_FIELD_NAME} field found".to_owned(), + }))?; + let app_id = data + .downcast_array_ref::() + .ok_or(StreamError::ChunkError(re_chunk::ChunkError::Malformed { + reason: format!( + "{CATALOG_APP_ID_FIELD_NAME} must be a utf8 array: {:?}", + tc.schema_ref() + ), + }))? + .value(0); + + let (_field, data) = tc + .components() + .find(|(f, _)| f.name() == CATALOG_START_TIME_FIELD_NAME) + .ok_or(StreamError::ChunkError(re_chunk::ChunkError::Malformed { + reason: "no {CATALOG_START_TIME_FIELD_NAME}} field found".to_owned(), + }))?; + let start_time = data + .downcast_array_ref::() + .ok_or(StreamError::ChunkError(re_chunk::ChunkError::Malformed { + reason: format!( + "{CATALOG_START_TIME_FIELD_NAME} must be a Timestamp array: {:?}", + tc.schema_ref() + ), + }))? + .value(0); + + Ok(StoreInfo { + application_id: ApplicationId::from(app_id), + store_id: store_id.clone(), + cloned_from: None, + is_official_example: false, + started: Time::from_ns_since_epoch(start_time), + store_source: StoreSource::Unknown, + store_version: None, + }) +} + +async fn stream_catalog_async( + tx: re_smart_channel::Sender, + redap_endpoint: Url, + on_msg: Option>, +) -> Result<(), StreamError> { + use tokio_stream::StreamExt as _; + + re_log::debug!("Connecting to {redap_endpoint}…"); + let mut client = { + #[cfg(target_arch = "wasm32")] + let tonic_client = tonic_web_wasm_client::Client::new_with_options( + redap_endpoint.to_string(), + tonic_web_wasm_client::options::FetchOptions::new(), + ); + + #[cfg(not(target_arch = "wasm32"))] + let tonic_client = tonic::transport::Endpoint::new(redap_endpoint.to_string())? + .connect() + .await?; + + StorageNodeClient::new(tonic_client) + }; + + re_log::debug!("Fetching catalog…"); + + let mut resp = client + .query_catalog(QueryCatalogRequest { + column_projection: None, // fetch all columns + filter: None, // fetch all rows + }) + .await + .map_err(TonicStatusError)? + .into_inner() + .map(|resp| { + resp.and_then(|r| { + r.decode() + .map_err(|err| tonic::Status::internal(err.to_string())) + }) + }); + + drop(client); + + if activate_catalog_blueprint(&tx).is_err() { + re_log::debug!("Failed to activate catalog blueprint"); + return Ok(()); + } + + // Craft the StoreInfo for the actual catalog data + let store_id = StoreId::from_string(StoreKind::Recording, CATALOG_REC_STORE_ID.to_owned()); + + let store_info = StoreInfo { + application_id: ApplicationId::from(CATALOG_APPLICATION_ID), + store_id: store_id.clone(), + cloned_from: None, + is_official_example: false, + started: Time::now(), + store_source: StoreSource::Unknown, + store_version: None, + }; + + if tx + .send(LogMsg::SetStoreInfo(SetStoreInfo { + row_id: *re_chunk::RowId::new(), + info: store_info, + })) + .is_err() + { + re_log::debug!("Receiver disconnected"); + return Ok(()); + } + + re_log::info!("Starting to read..."); + while let Some(result) = resp.next().await { + let input = TransportChunk::from(result.map_err(TonicStatusError)?); + + // Catalog received from the ReDap server isn't suitable for direct conversion to a Rerun Chunk: + // - conversion expects "data" columns to be ListArrays, hence we need to convert any individual row column data to ListArray + // - conversion expects the input TransportChunk to have a ChunkId so we need to add that piece of metadata + + let mut fields: Vec = Vec::new(); + let mut columns: Vec = Vec::new(); + // add the (row id) control field + let (row_id_field, row_id_data) = input.controls().next().ok_or( + StreamError::ChunkError(re_chunk::ChunkError::Malformed { + reason: "no control field found".to_owned(), + }), + )?; + + fields.push( + ArrowField::new( + RowId::name().to_string(), // need to rename to Rerun Chunk expected control field + row_id_field.data_type().clone(), + false, /* not nullable */ + ) + .with_metadata(TransportChunk::field_metadata_control_column()), + ); + columns.push(row_id_data.clone()); + + // next add any timeline field + for (field, data) in input.timelines() { + fields.push(field.clone()); + columns.push(data.clone()); + } + + // now add all the 'data' fields - we slice each column array into individual arrays and then convert the whole lot into a ListArray + for (field, data) in input.components() { + let data_field_inner = + ArrowField::new("item", field.data_type().clone(), true /* nullable */); + + let data_field = ArrowField::new( + field.name().clone(), + ArrowDataType::List(Arc::new(data_field_inner.clone())), + false, /* not nullable */ + ) + .with_metadata(TransportChunk::field_metadata_data_column()); + + let mut sliced: Vec = Vec::new(); + for idx in 0..data.len() { + sliced.push(data.clone().slice(idx, 1)); + } + + let data_arrays = sliced.iter().map(|e| Some(e.as_ref())).collect::>(); + #[allow(clippy::unwrap_used)] // we know we've given the right field type + let data_field_array: arrow::array::ListArray = + re_arrow_util::arrow_util::arrays_to_list_array( + data_field_inner.data_type().clone(), + &data_arrays, + ) + .unwrap(); + + fields.push(data_field); + columns.push(as_array_ref(data_field_array)); + } + + let schema = { + let metadata = HashMap::from_iter([ + ( + TransportChunk::CHUNK_METADATA_KEY_ENTITY_PATH.to_owned(), + "catalog".to_owned(), + ), + ( + TransportChunk::CHUNK_METADATA_KEY_ID.to_owned(), + ChunkId::new().to_string(), + ), + ]); + arrow::datatypes::Schema::new_with_metadata(fields, metadata) + }; + + let record_batch = ArrowRecordBatch::try_new(schema.into(), columns) + .map_err(re_chunk::ChunkError::from)?; + let mut chunk = Chunk::from_record_batch(record_batch)?; + + // finally, enrich catalog data with RecordingUri that's based on the ReDap endpoint (that we know) + // and the recording id (that we have in the catalog data) + let host = redap_endpoint + .host() + .ok_or(StreamError::InvalidUri(format!( + "couldn't get host from {redap_endpoint}" + )))?; + let port = redap_endpoint + .port() + .ok_or(StreamError::InvalidUri(format!( + "couldn't get port from {redap_endpoint}" + )))?; + + let recording_uri_arrays: Vec = chunk + .iter_slices::(CATALOG_ID_FIELD_NAME.into()) + .map(|id| { + let rec_id = &id[0]; // each component batch is of length 1 i.e. single 'id' value + + let recording_uri = format!("rerun://{host}:{port}/recording/{rec_id}"); + + as_array_ref(ArrowStringArray::from(vec![recording_uri])) + }) + .collect(); + + let recording_id_arrays = recording_uri_arrays + .iter() + .map(|e| Some(e.as_ref())) + .collect::>(); + + let rec_id_field = ArrowField::new("item", ArrowDataType::Utf8, true); + #[allow(clippy::unwrap_used)] // we know we've given the right field type + let uris = re_arrow_util::arrow_util::arrays_to_list_array( + rec_id_field.data_type().clone(), + &recording_id_arrays, + ) + .unwrap(); + + chunk.add_component(ComponentDescriptor::new(RecordingUri::name()), uris)?; + + if tx + .send(LogMsg::ArrowMsg(store_id.clone(), chunk.to_arrow_msg()?)) + .is_err() + { + re_log::debug!("Receiver disconnected"); + return Ok(()); + } + + if let Some(on_msg) = &on_msg { + on_msg(); + } + } + + Ok(()) +} + +// Craft a blueprint from relevant chunks and activate it +// TODO(zehiko) - manual crafting of the blueprint as we have below will go away and be replaced +// by either a blueprint crafted using rust Blueprint API or a blueprint fetched from ReDap (#8470) +fn activate_catalog_blueprint( + tx: &re_smart_channel::Sender, +) -> Result<(), Box> { + let blueprint_store_id = + StoreId::from_string(StoreKind::Blueprint, CATALOG_BP_STORE_ID.to_owned()); + let blueprint_store_info = StoreInfo { + application_id: ApplicationId::from(CATALOG_APPLICATION_ID), + store_id: blueprint_store_id.clone(), + cloned_from: None, + is_official_example: false, + started: Time::now(), + store_source: StoreSource::Unknown, + store_version: None, + }; + + if tx + .send(LogMsg::SetStoreInfo(SetStoreInfo { + row_id: *re_chunk::RowId::new(), + info: blueprint_store_info, + })) + .is_err() + { + re_log::debug!("Receiver disconnected"); + return Ok(()); + } + + let timepoint = [(Timeline::new_sequence("blueprint"), 1)]; + + let vb = ViewBlueprint::new("Dataframe") + .with_visible(true) + .with_space_origin("/"); + + // TODO(zehiko) we shouldn't really be creating all these ids and entity paths manually... (#8470) + let view_uuid = uuid::Uuid::new_v4(); + let view_entity_path = format!("/view/{view_uuid}"); + let view_chunk = ChunkBuilder::new(ChunkId::new(), view_entity_path.clone().into()) + .with_archetype(RowId::new(), timepoint, &vb) + .build()?; + + let epf = EntityPathFilter::parse_forgiving("/**"); + let vc = ViewContents::new(epf.iter_expressions()); + let view_contents_chunk = ChunkBuilder::new( + ChunkId::new(), + format!( + "{}/{}", + view_entity_path.clone(), + ViewContents::name().short_name() + ) + .into(), + ) + .with_archetype(RowId::new(), timepoint, &vc) + .build()?; + + let rc = ContainerBlueprint::new(ContainerKind::Grid) + .with_contents(&[EntityPath::from(view_entity_path)]) + .with_visible(true); + + let container_uuid = uuid::Uuid::new_v4(); + let container_chunk = ChunkBuilder::new( + ChunkId::new(), + format!("/container/{container_uuid}").into(), + ) + .with_archetype(RowId::new(), timepoint, &rc) + .build()?; + + let vp = ViewportBlueprint::new().with_root_container(RootContainer(container_uuid.into())); + let viewport_chunk = ChunkBuilder::new(ChunkId::new(), "/viewport".into()) + .with_archetype(RowId::new(), timepoint, &vp) + .build()?; + + for chunk in &[ + view_chunk, + view_contents_chunk, + container_chunk, + viewport_chunk, + ] { + if tx + .send(LogMsg::ArrowMsg( + blueprint_store_id.clone(), + chunk.to_arrow_msg()?, + )) + .is_err() + { + re_log::debug!("Receiver disconnected"); + return Ok(()); + } + } + + let blueprint_activation = BlueprintActivationCommand { + blueprint_id: blueprint_store_id.clone(), + make_active: true, + make_default: true, + }; + + if tx + .send(LogMsg::BlueprintActivationCommand(blueprint_activation)) + .is_err() + { + re_log::debug!("Receiver disconnected"); + return Ok(()); + } + + Ok(()) +} diff --git a/crates/top/re_sdk/Cargo.toml b/crates/top/re_sdk/Cargo.toml index d0b75cb3a8f8..c427125fd155 100644 --- a/crates/top/re_sdk/Cargo.toml +++ b/crates/top/re_sdk/Cargo.toml @@ -47,13 +47,14 @@ web_viewer = [ "re_ws_comms?/server", ] -grpc = ["dep:re_grpc_client"] +grpc = ["re_grpc_client/redap"] [dependencies] re_build_info.workspace = true re_byte_size.workspace = true re_chunk.workspace = true +re_grpc_client.workspace = true re_log_encoding = { workspace = true, features = ["encoder"] } re_log_types.workspace = true re_log.workspace = true @@ -73,7 +74,6 @@ thiserror.workspace = true # Optional dependencies re_data_loader = { workspace = true, optional = true } -re_grpc_client = { workspace = true, optional = true } re_smart_channel = { workspace = true, optional = true } re_ws_comms = { workspace = true, optional = true } re_web_viewer_server = { workspace = true, optional = true } diff --git a/crates/viewer/re_viewer/Cargo.toml b/crates/viewer/re_viewer/Cargo.toml index f863eb8ed24c..150edd40a2c2 100644 --- a/crates/viewer/re_viewer/Cargo.toml +++ b/crates/viewer/re_viewer/Cargo.toml @@ -41,7 +41,7 @@ analytics = ["dep:re_analytics"] map_view = ["dep:re_view_map"] ## Enable the gRPC Rerun Data Platform data source. -grpc = ["re_data_source/grpc", "dep:re_grpc_client"] +grpc = ["re_data_source/grpc", "re_grpc_client/redap"] [dependencies] @@ -59,6 +59,7 @@ re_chunk_store_ui.workspace = true re_entity_db.workspace = true re_error.workspace = true re_format.workspace = true +re_grpc_client.workspace = true re_log = { workspace = true, features = ["setup"] } re_log_encoding = { workspace = true, features = [ "decoder", @@ -93,7 +94,6 @@ re_ws_comms = { workspace = true, features = ["client"] } # Internal (optional): re_analytics = { workspace = true, optional = true } -re_grpc_client = { workspace = true, optional = true } re_view_map = { workspace = true, optional = true } diff --git a/crates/viewer/re_viewer/src/web_tools.rs b/crates/viewer/re_viewer/src/web_tools.rs index 8e2e1283e8b0..259f6173a0eb 100644 --- a/crates/viewer/re_viewer/src/web_tools.rs +++ b/crates/viewer/re_viewer/src/web_tools.rs @@ -94,7 +94,6 @@ enum EndpointCategory { /// An eventListener for rrd posted from containing html WebEventListener(String), - #[cfg(feature = "grpc")] /// A stream of messages over gRPC, relayed from the SDK. MessageProxy(String), } @@ -111,14 +110,7 @@ impl EndpointCategory { Self::WebEventListener(uri) } else if uri.starts_with("temp:") { // TODO(#8761): URL prefix - #[cfg(feature = "grpc")] - { - Self::MessageProxy(uri) - } - #[cfg(not(feature = "grpc"))] - { - panic!("Required the 'grpc' feature flag to be enabled"); - } + Self::MessageProxy(uri) } else { // If this is something like `foo.com` we can't know what it is until we connect to it. // We could/should connect and see what it is, but for now we just take a wild guess instead: @@ -154,7 +146,7 @@ pub fn url_to_receiver( #[cfg(feature = "grpc")] EndpointCategory::RerunGrpc(url) => { - re_grpc_client::stream_from_redap(url, Some(ui_waker)).map_err(|err| err.into()) + re_grpc_client::redap::stream_from_redap(url, Some(ui_waker)).map_err(|err| err.into()) } #[cfg(not(feature = "grpc"))] EndpointCategory::RerunGrpc(_url) => { @@ -196,7 +188,6 @@ pub fn url_to_receiver( EndpointCategory::WebSocket(url) => re_data_source::connect_to_ws_url(&url, Some(ui_waker)) .with_context(|| format!("Failed to connect to WebSocket server at {url}.")), - #[cfg(feature = "grpc")] EndpointCategory::MessageProxy(url) => { re_grpc_client::message_proxy::read::stream(url, Some(ui_waker)) .map_err(|err| err.into()) diff --git a/rerun_py/src/remote.rs b/rerun_py/src/remote.rs index dd0763ac8e0c..12eeebfa04af 100644 --- a/rerun_py/src/remote.rs +++ b/rerun_py/src/remote.rs @@ -130,7 +130,7 @@ impl PyStorageNodeClient { )); } - re_grpc_client::store_info_from_catalog_chunk( + re_grpc_client::redap::store_info_from_catalog_chunk( &re_chunk::TransportChunk::from(resp[0].clone()), id, ) From 086595cc76ccb5453734f8f49df5c7b61551c1cb Mon Sep 17 00:00:00 2001 From: jprochazk Date: Mon, 27 Jan 2025 11:22:23 +0100 Subject: [PATCH 02/87] send stream error through smart channel --- crates/store/re_data_source/src/data_source.rs | 2 +- .../re_grpc_client/src/message_proxy/read.rs | 16 +++++++--------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/crates/store/re_data_source/src/data_source.rs b/crates/store/re_data_source/src/data_source.rs index f3a14b0a57eb..26835eb73f20 100644 --- a/crates/store/re_data_source/src/data_source.rs +++ b/crates/store/re_data_source/src/data_source.rs @@ -260,7 +260,7 @@ impl DataSource { } Self::MessageProxy { url } => { - re_grpc_client::message_proxy::stream(url, on_msg).map_err(|err| err.into()) + re_grpc_client::message_proxy::stream(&url, on_msg).map_err(|err| err.into()) } } } diff --git a/crates/store/re_grpc_client/src/message_proxy/read.rs b/crates/store/re_grpc_client/src/message_proxy/read.rs index b1a0d9241b25..ce8ef23de4bf 100644 --- a/crates/store/re_grpc_client/src/message_proxy/read.rs +++ b/crates/store/re_grpc_client/src/message_proxy/read.rs @@ -11,24 +11,22 @@ use crate::StreamError; use crate::TonicStatusError; pub fn stream( - url: String, + url: &str, on_msg: Option>, ) -> Result, InvalidMessageProxyAddress> { re_log::debug!("Loading {url} via gRPC…"); - let parsed_url = MessageProxyAddress::parse(&url)?; + let parsed_url = MessageProxyAddress::parse(url)?; + let url = url.to_owned(); let (tx, rx) = re_smart_channel::smart_channel( re_smart_channel::SmartMessageSource::MessageProxy { url: url.clone() }, - re_smart_channel::SmartChannelSource::MessageProxy { url: url.clone() }, + re_smart_channel::SmartChannelSource::MessageProxy { url }, ); crate::spawn_future(async move { - if let Err(err) = stream_async(parsed_url, tx, on_msg).await { - re_log::error!( - "Error while streaming from {url}: {}", - re_error::format_ref(&err) - ); + if let Err(err) = stream_async(parsed_url, &tx, on_msg).await { + tx.quit(Some(Box::new(err))).ok(); } }); @@ -80,7 +78,7 @@ pub struct InvalidMessageProxyAddress { async fn stream_async( url: MessageProxyAddress, - tx: re_smart_channel::Sender, + tx: &re_smart_channel::Sender, on_msg: Option>, ) -> Result<(), StreamError> { let mut client = { From daf8278fd8466d55cdd75eca164311f7dc0900d2 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Mon, 27 Jan 2025 11:22:39 +0100 Subject: [PATCH 03/87] expose grpc client/server in re_sdk --- Cargo.lock | 1 + Cargo.toml | 1 + crates/store/re_grpc_server/src/lib.rs | 53 +++++++++++++++++++++++++ crates/store/re_grpc_server/src/main.rs | 47 +--------------------- crates/top/re_sdk/Cargo.toml | 1 + crates/top/re_sdk/src/lib.rs | 2 + 6 files changed, 60 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 53ebfcc9378d..15ca5efe35d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6400,6 +6400,7 @@ dependencies = [ "re_chunk_store", "re_data_loader", "re_grpc_client", + "re_grpc_server", "re_log", "re_log_encoding", "re_log_types", diff --git a/Cargo.toml b/Cargo.toml index cd9451fa1177..f28ba216dc5b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,7 @@ re_dataframe = { path = "crates/store/re_dataframe", version = "=0.22.0-alpha.1" re_entity_db = { path = "crates/store/re_entity_db", version = "=0.22.0-alpha.1", default-features = false } re_format_arrow = { path = "crates/store/re_format_arrow", version = "=0.22.0-alpha.1", default-features = false } re_grpc_client = { path = "crates/store/re_grpc_client", version = "=0.22.0-alpha.1", default-features = false } +re_grpc_server = { path = "crates/store/re_grpc_server", version = "=0.22.0-alpha.1", default-features = false } re_protos = { path = "crates/store/re_protos", version = "=0.22.0-alpha.1", default-features = false } re_log_encoding = { path = "crates/store/re_log_encoding", version = "=0.22.0-alpha.1", default-features = false } re_log_types = { path = "crates/store/re_log_types", version = "=0.22.0-alpha.1", default-features = false } diff --git a/crates/store/re_grpc_server/src/lib.rs b/crates/store/re_grpc_server/src/lib.rs index 020ba2a4559e..d7fb17ac3027 100644 --- a/crates/store/re_grpc_server/src/lib.rs +++ b/crates/store/re_grpc_server/src/lib.rs @@ -1,6 +1,9 @@ //! Server implementation of an in-memory Storage Node. use std::collections::VecDeque; +use std::net::IpAddr; +use std::net::Ipv4Addr; +use std::net::SocketAddr; use std::pin::Pin; use re_byte_size::SizeBytes; @@ -9,12 +12,62 @@ use re_protos::{ log_msg::v0::LogMsg as LogMsgProto, sdk_comms::v0::{message_proxy_server, Empty}, }; +use tokio::net::TcpListener; use tokio::sync::broadcast; use tokio::sync::mpsc; use tokio::sync::oneshot; use tokio_stream::wrappers::BroadcastStream; use tokio_stream::Stream; use tokio_stream::StreamExt as _; +use tonic::transport::server::TcpIncoming; +use tonic::transport::Server; + +pub const DEFAULT_GRPC_PORT: u16 = 1852; +pub const DEFAULT_GRPC_ADDR: SocketAddr = + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), DEFAULT_GRPC_PORT); +pub const DEFAULT_MEMORY_LIMIT: MemoryLimit = MemoryLimit::UNLIMITED; + +/// Listen for incoming clients on `addr`. +/// +/// The server runs on the current task. +pub async fn serve( + addr: SocketAddr, + memory_limit: MemoryLimit, +) -> Result<(), tonic::transport::Error> { + let tcp_listener = TcpListener::bind(addr) + .await + .unwrap_or_else(|err| panic!("failed to bind listener on {addr}: {err}")); + let incoming = + TcpIncoming::from_listener(tcp_listener, true, None).expect("failed to init listener"); + + re_log::info!("Listening for gRPC connections on {addr}"); + + use tower_http::cors::{Any, CorsLayer}; + let cors = CorsLayer::new() + .allow_origin(Any) + .allow_methods(Any) + .allow_headers(Any); + + let grpc_web = tonic_web::GrpcWebLayer::new(); + + let routes = { + let mut routes_builder = tonic::service::Routes::builder(); + routes_builder.add_service( + re_protos::sdk_comms::v0::message_proxy_server::MessageProxyServer::new( + MessageProxy::new(memory_limit), + ), + ); + routes_builder.routes() + }; + + Server::builder() + .accept_http1(true) // Support `grpc-web` clients + .layer(cors) // Allow CORS requests from web clients + .layer(grpc_web) // Support `grpc-web` clients + .add_routes(routes) + .serve_with_incoming(incoming) + .await +} enum Event { /// New client connected, requesting full history and subscribing to new messages. diff --git a/crates/store/re_grpc_server/src/main.rs b/crates/store/re_grpc_server/src/main.rs index cf99f3c3e3be..d17c5849bd75 100644 --- a/crates/store/re_grpc_server/src/main.rs +++ b/crates/store/re_grpc_server/src/main.rs @@ -1,51 +1,8 @@ -use std::net::IpAddr; -use std::net::Ipv4Addr; -use std::net::SocketAddr; - -use re_grpc_server::MessageProxy; -use re_memory::MemoryLimit; -use re_protos::sdk_comms::v0::message_proxy_server::MessageProxyServer; -use tokio::net::TcpListener; -use tonic::transport::server::TcpIncoming; -use tonic::transport::Server; - -const DEFAULT_GRPC_PORT: u16 = 1852; -const DEFAULT_GRPC_ADDR: SocketAddr = - SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), DEFAULT_GRPC_PORT); +use re_grpc_server::{serve, DEFAULT_GRPC_ADDR, DEFAULT_MEMORY_LIMIT}; #[tokio::main(flavor = "current_thread")] async fn main() -> Result<(), tonic::transport::Error> { re_log::setup_logging(); - let tcp_listener = TcpListener::bind(DEFAULT_GRPC_ADDR) - .await - .unwrap_or_else(|err| panic!("failed to bind listener on {DEFAULT_GRPC_ADDR}: {err}")); - let incoming = - TcpIncoming::from_listener(tcp_listener, true, None).expect("failed to init listener"); - - re_log::info!("Listening for gRPC connections on {DEFAULT_GRPC_ADDR}"); - - use tower_http::cors::{Any, CorsLayer}; - let cors = CorsLayer::new() - .allow_origin(Any) - .allow_methods(Any) - .allow_headers(Any); - - let grpc_web = tonic_web::GrpcWebLayer::new(); - - let routes = { - let mut routes_builder = tonic::service::Routes::builder(); - routes_builder.add_service(MessageProxyServer::new(MessageProxy::new( - MemoryLimit::UNLIMITED, - ))); - routes_builder.routes() - }; - - Server::builder() - .accept_http1(true) // Support `grpc-web` clients - .layer(cors) // Allow CORS requests from web clients - .layer(grpc_web) // Support `grpc-web` clients - .add_routes(routes) - .serve_with_incoming(incoming) - .await + serve(DEFAULT_GRPC_ADDR, DEFAULT_MEMORY_LIMIT).await } diff --git a/crates/top/re_sdk/Cargo.toml b/crates/top/re_sdk/Cargo.toml index c427125fd155..12583925c34d 100644 --- a/crates/top/re_sdk/Cargo.toml +++ b/crates/top/re_sdk/Cargo.toml @@ -55,6 +55,7 @@ re_build_info.workspace = true re_byte_size.workspace = true re_chunk.workspace = true re_grpc_client.workspace = true +re_grpc_server.workspace = true re_log_encoding = { workspace = true, features = ["encoder"] } re_log_types.workspace = true re_log.workspace = true diff --git a/crates/top/re_sdk/src/lib.rs b/crates/top/re_sdk/src/lib.rs index f353c8dc7201..65bc8dc0487f 100644 --- a/crates/top/re_sdk/src/lib.rs +++ b/crates/top/re_sdk/src/lib.rs @@ -112,6 +112,8 @@ pub mod web_viewer; /// Re-exports of other crates. pub mod external { + pub use re_grpc_client; + pub use re_grpc_server; pub use re_log; pub use re_log_encoding; pub use re_log_types; From f10370cfca6f1471e65e2bd40f8c62d57743b228 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Mon, 27 Jan 2025 16:05:14 +0100 Subject: [PATCH 04/87] Refactor `re_grpc_server` serve impl --- crates/store/re_grpc_server/Cargo.toml | 1 + crates/store/re_grpc_server/src/lib.rs | 112 ++++++++++++++++++------ crates/store/re_grpc_server/src/main.rs | 4 +- 3 files changed, 90 insertions(+), 27 deletions(-) diff --git a/crates/store/re_grpc_server/Cargo.toml b/crates/store/re_grpc_server/Cargo.toml index 538b56257975..1d868cd7bcb7 100644 --- a/crates/store/re_grpc_server/Cargo.toml +++ b/crates/store/re_grpc_server/Cargo.toml @@ -29,6 +29,7 @@ re_log_encoding = { workspace = true, features = ["encoder", "decoder"] } re_log_types.workspace = true re_memory.workspace = true re_protos.workspace = true +re_smart_channel.workspace = true re_tracing.workspace = true re_types.workspace = true diff --git a/crates/store/re_grpc_server/src/lib.rs b/crates/store/re_grpc_server/src/lib.rs index d7fb17ac3027..1d362da814ba 100644 --- a/crates/store/re_grpc_server/src/lib.rs +++ b/crates/store/re_grpc_server/src/lib.rs @@ -1,9 +1,9 @@ //! Server implementation of an in-memory Storage Node. use std::collections::VecDeque; -use std::net::IpAddr; use std::net::Ipv4Addr; use std::net::SocketAddr; +use std::net::SocketAddrV4; use std::pin::Pin; use re_byte_size::SizeBytes; @@ -22,25 +22,28 @@ use tokio_stream::StreamExt as _; use tonic::transport::server::TcpIncoming; use tonic::transport::Server; -pub const DEFAULT_GRPC_PORT: u16 = 1852; -pub const DEFAULT_GRPC_ADDR: SocketAddr = - SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), DEFAULT_GRPC_PORT); +pub const DEFAULT_SERVER_PORT: u16 = 1852; pub const DEFAULT_MEMORY_LIMIT: MemoryLimit = MemoryLimit::UNLIMITED; /// Listen for incoming clients on `addr`. /// /// The server runs on the current task. -pub async fn serve( - addr: SocketAddr, - memory_limit: MemoryLimit, -) -> Result<(), tonic::transport::Error> { - let tcp_listener = TcpListener::bind(addr) - .await - .unwrap_or_else(|err| panic!("failed to bind listener on {addr}: {err}")); +pub async fn serve(port: u16, memory_limit: MemoryLimit) -> Result<(), tonic::transport::Error> { + serve_impl(port, MessageProxy::new(memory_limit)).await +} + +async fn serve_impl(port: u16, message_proxy: MessageProxy) -> Result<(), tonic::transport::Error> { + let tcp_listener = TcpListener::bind(SocketAddr::V4(SocketAddrV4::new( + Ipv4Addr::new(0, 0, 0, 0), + port, + ))) + .await + .unwrap_or_else(|err| panic!("failed to bind listener on port {port}: {err}")); + let incoming = TcpIncoming::from_listener(tcp_listener, true, None).expect("failed to init listener"); - re_log::info!("Listening for gRPC connections on {addr}"); + re_log::info!("Listening for gRPC connections on http://0.0.0.0:{port}"); use tower_http::cors::{Any, CorsLayer}; let cors = CorsLayer::new() @@ -53,9 +56,7 @@ pub async fn serve( let routes = { let mut routes_builder = tonic::service::Routes::builder(); routes_builder.add_service( - re_protos::sdk_comms::v0::message_proxy_server::MessageProxyServer::new( - MessageProxy::new(memory_limit), - ), + re_protos::sdk_comms::v0::message_proxy_server::MessageProxyServer::new(message_proxy), ); routes_builder.routes() }; @@ -69,6 +70,55 @@ pub async fn serve( .await } +pub fn spawn_with_recv( + port: u16, + memory_limit: MemoryLimit, +) -> re_smart_channel::Receiver { + let (channel_tx, channel_rx) = re_smart_channel::smart_channel( + re_smart_channel::SmartMessageSource::MessageProxy { + url: format!("http://localhost:{port}"), + }, + re_smart_channel::SmartChannelSource::MessageProxy { + url: format!("http://localhost:{port}"), + }, + ); + let (message_proxy, mut broadcast_rx) = MessageProxy::new_with_recv(memory_limit); + tokio::spawn(serve_impl(port, message_proxy)); + tokio::spawn(async move { + loop { + let msg = match broadcast_rx.recv().await { + Ok(msg) => re_log_encoding::protobuf_conversions::log_msg_from_proto(msg), + Err(broadcast::error::RecvError::Closed) => { + re_log::debug!("message proxy server shut down, closing receiver"); + channel_tx.quit(None).ok(); + break; + } + Err(broadcast::error::RecvError::Lagged(n)) => { + re_log::debug!( + "message proxy receiver dropped {n} messages due to backpressure" + ); + continue; + } + }; + match msg { + Ok(msg) => { + if channel_tx.send(msg).is_err() { + re_log::debug!( + "message proxy smart channel receiver closed, closing sender" + ); + break; + } + } + Err(err) => { + re_log::error!("dropping LogMsg due to failed decode: {err}"); + continue; + } + } + } + }); + channel_rx +} + enum Event { /// New client connected, requesting full history and subscribing to new messages. NewClient(oneshot::Sender<(Vec, broadcast::Receiver)>), @@ -100,12 +150,14 @@ struct EventLoop { } impl EventLoop { - fn new(server_memory_limit: MemoryLimit, event_rx: mpsc::Receiver) -> Self { + fn new( + server_memory_limit: MemoryLimit, + event_rx: mpsc::Receiver, + broadcast_tx: broadcast::Sender, + ) -> Self { Self { server_memory_limit, - // Channel capacity is completely arbitrary. - // We just want enough capacity to handle bursts of messages. - broadcast_tx: broadcast::channel(1024).0, + broadcast_tx, event_rx, ordered_message_queue: Default::default(), ordered_message_bytes: 0, @@ -228,20 +280,30 @@ pub struct MessageProxy { impl MessageProxy { pub fn new(server_memory_limit: MemoryLimit) -> Self { + Self::new_with_recv(server_memory_limit).0 + } + + pub fn new_with_recv( + server_memory_limit: MemoryLimit, + ) -> (Self, broadcast::Receiver) { // Channel capacity is completely arbitrary. // We just want something large enough to handle bursts of messages. let (event_tx, event_rx) = mpsc::channel(1024); + let (broadcast_tx, broadcast_rx) = broadcast::channel(1024); let task_handle = tokio::spawn(async move { - EventLoop::new(server_memory_limit, event_rx) + EventLoop::new(server_memory_limit, event_rx, broadcast_tx) .run_in_place() .await; }); - Self { - _queue_task_handle: task_handle, - event_tx, - } + ( + Self { + _queue_task_handle: task_handle, + event_tx, + }, + broadcast_rx, + ) } async fn push(&self, msg: LogMsgProto) { @@ -270,7 +332,7 @@ impl MessageProxy { }) }); - Box::pin(history.merge(channel)) + Box::pin(history.chain(channel)) } } diff --git a/crates/store/re_grpc_server/src/main.rs b/crates/store/re_grpc_server/src/main.rs index d17c5849bd75..58dfe287b02f 100644 --- a/crates/store/re_grpc_server/src/main.rs +++ b/crates/store/re_grpc_server/src/main.rs @@ -1,8 +1,8 @@ -use re_grpc_server::{serve, DEFAULT_GRPC_ADDR, DEFAULT_MEMORY_LIMIT}; +use re_grpc_server::{serve, DEFAULT_MEMORY_LIMIT, DEFAULT_SERVER_PORT}; #[tokio::main(flavor = "current_thread")] async fn main() -> Result<(), tonic::transport::Error> { re_log::setup_logging(); - serve(DEFAULT_GRPC_ADDR, DEFAULT_MEMORY_LIMIT).await + serve(DEFAULT_SERVER_PORT, DEFAULT_MEMORY_LIMIT).await } From c46af8bb94eb189668fed593e584554923fb31fa Mon Sep 17 00:00:00 2001 From: jprochazk Date: Mon, 27 Jan 2025 16:09:01 +0100 Subject: [PATCH 05/87] Remove feature gate on `re_sdk::GrpcSink` --- .../re_grpc_client/src/message_proxy/write.rs | 1 + crates/top/re_sdk/src/lib.rs | 3 +- crates/top/re_sdk/src/log_sink.rs | 56 +++++++++---------- 3 files changed, 27 insertions(+), 33 deletions(-) diff --git a/crates/store/re_grpc_client/src/message_proxy/write.rs b/crates/store/re_grpc_client/src/message_proxy/write.rs index da3d8fddf283..e7bbafa28abe 100644 --- a/crates/store/re_grpc_client/src/message_proxy/write.rs +++ b/crates/store/re_grpc_client/src/message_proxy/write.rs @@ -38,6 +38,7 @@ pub struct Client { } impl Client { + /// `url` should be the `http://` or `https://` URL of the server. #[expect(clippy::needless_pass_by_value)] pub fn new(url: impl Into, options: Options) -> Self { let url: String = url.into(); diff --git a/crates/top/re_sdk/src/lib.rs b/crates/top/re_sdk/src/lib.rs index 65bc8dc0487f..954835afc2d6 100644 --- a/crates/top/re_sdk/src/lib.rs +++ b/crates/top/re_sdk/src/lib.rs @@ -70,8 +70,7 @@ pub mod sink { BufferedSink, CallbackSink, LogSink, MemorySink, MemorySinkStorage, TcpSink, }; - #[cfg(feature = "grpc")] - pub use crate::log_sink::grpc::GrpcSink; + pub use crate::log_sink::GrpcSink; #[cfg(not(target_arch = "wasm32"))] pub use re_log_encoding::{FileSink, FileSinkError}; diff --git a/crates/top/re_sdk/src/log_sink.rs b/crates/top/re_sdk/src/log_sink.rs index a455399d9ea2..048408b74fdc 100644 --- a/crates/top/re_sdk/src/log_sink.rs +++ b/crates/top/re_sdk/src/log_sink.rs @@ -2,6 +2,7 @@ use std::fmt; use std::sync::Arc; use parking_lot::Mutex; +use re_grpc_client::message_proxy::write::Client as MessageProxyClient; use re_log_encoding::encoder::encode_as_bytes_local; use re_log_encoding::encoder::{local_raw_encoder, EncodeError}; use re_log_types::{BlueprintActivationCommand, LogMsg, StoreId}; @@ -368,40 +369,33 @@ impl LogSink for TcpSink { } } -#[cfg(feature = "grpc")] -pub mod grpc { - use super::LogSink; - use re_grpc_client::message_proxy::write::Client; - use re_log_types::LogMsg; - - /// Stream log messages to an in-memory storage node. - pub struct GrpcSink { - client: Client, - } - - impl GrpcSink { - /// Connect to the in-memory storage node over HTTP. - /// - /// ### Example - /// - /// ```ignore - /// GrpcSink::new("http://127.0.0.1:9434"); - /// ``` - #[inline] - pub fn new(addr: impl Into) -> Self { - Self { - client: Client::new(addr, Default::default()), - } +/// Stream log messages to an in-memory storage node. +pub struct GrpcSink { + client: MessageProxyClient, +} + +impl GrpcSink { + /// Connect to the in-memory storage node over HTTP. + /// + /// ### Example + /// + /// ```ignore + /// GrpcSink::new("http://127.0.0.1:9434"); + /// ``` + #[inline] + pub fn new(addr: impl Into) -> Self { + Self { + client: MessageProxyClient::new(addr, Default::default()), } } +} - impl LogSink for GrpcSink { - fn send(&self, msg: LogMsg) { - self.client.send(msg); - } +impl LogSink for GrpcSink { + fn send(&self, msg: LogMsg) { + self.client.send(msg); + } - fn flush_blocking(&self) { - self.client.flush(); - } + fn flush_blocking(&self) { + self.client.flush(); } } From 459d12d498fdbc0c7ecfd005548bfab99843cdd0 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Mon, 27 Jan 2025 16:18:54 +0100 Subject: [PATCH 06/87] Ensure `tokio` is always available --- Cargo.lock | 3 +++ crates/top/rerun-cli/Cargo.toml | 5 ++--- crates/top/rerun-cli/src/bin/rerun.rs | 10 --------- crates/top/rerun/Cargo.toml | 4 +++- crates/top/rerun/src/commands/entrypoint.rs | 25 ++++++++++++++++++++- 5 files changed, 32 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 15ca5efe35d6..dbf94a3936e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6072,6 +6072,7 @@ dependencies = [ "re_log_types", "re_memory", "re_protos", + "re_smart_channel", "re_tracing", "re_types", "tokio", @@ -7361,6 +7362,7 @@ dependencies = [ "re_error", "re_format", "re_format_arrow", + "re_grpc_server", "re_log", "re_log_encoding", "re_log_types", @@ -7375,6 +7377,7 @@ dependencies = [ "re_web_viewer_server", "re_ws_comms", "similar-asserts", + "tokio", "unindent", ] diff --git a/crates/top/rerun-cli/Cargo.toml b/crates/top/rerun-cli/Cargo.toml index 18e3f1778949..9f20966be467 100644 --- a/crates/top/rerun-cli/Cargo.toml +++ b/crates/top/rerun-cli/Cargo.toml @@ -73,7 +73,7 @@ native_viewer = ["rerun/native_viewer"] map_view = ["rerun/map_view"] ## Enable the gRPC Rerun Data Platform data source. -grpc = ["rerun/grpc", "dep:tokio"] +grpc = ["rerun/grpc"] ## Support serving a web viewer over HTTP. ## @@ -99,8 +99,7 @@ rerun = { workspace = true, features = [ document-features.workspace = true mimalloc = "0.1.43" - -tokio = { workspace = true, optional = true, features = [ +tokio = { workspace = true, features = [ "macros", "rt-multi-thread", ] } diff --git a/crates/top/rerun-cli/src/bin/rerun.rs b/crates/top/rerun-cli/src/bin/rerun.rs index dbe64430b010..022a7e4042b2 100644 --- a/crates/top/rerun-cli/src/bin/rerun.rs +++ b/crates/top/rerun-cli/src/bin/rerun.rs @@ -16,18 +16,8 @@ use re_memory::AccountingAllocator; static GLOBAL: AccountingAllocator = AccountingAllocator::new(mimalloc::MiMalloc); -#[cfg(feature = "grpc")] #[tokio::main] async fn main() -> std::process::ExitCode { - main_impl() -} - -#[cfg(not(feature = "grpc"))] -fn main() -> std::process::ExitCode { - main_impl() -} - -fn main_impl() -> std::process::ExitCode { let main_thread_token = rerun::MainThreadToken::i_promise_i_am_on_the_main_thread(); re_log::setup_logging(); diff --git a/crates/top/rerun/Cargo.toml b/crates/top/rerun/Cargo.toml index 974fddd12dcf..838054c664fd 100644 --- a/crates/top/rerun/Cargo.toml +++ b/crates/top/rerun/Cargo.toml @@ -106,7 +106,7 @@ run = [ ] ## Support for running a TCP server that listens to incoming log messages from a Rerun SDK. -server = ["re_sdk_comms?/server"] +server = ["re_sdk_comms?/server", "dep:re_grpc_server"] ## Embed the Rerun SDK & built-in types and re-export all of their public symbols. sdk = ["dep:re_sdk", "dep:re_types"] @@ -145,12 +145,14 @@ arrow.workspace = true document-features.workspace = true itertools.workspace = true similar-asserts.workspace = true +tokio = { workspace = true, features = ["rt-multi-thread"] } # Optional dependencies: re_analytics = { workspace = true, optional = true } re_chunk_store = { workspace = true, optional = true } re_data_source = { workspace = true, optional = true } re_dataframe = { workspace = true, optional = true } +re_grpc_server = { workspace = true, optional = true } re_sdk = { workspace = true, optional = true } re_sdk_comms = { workspace = true, optional = true } re_types = { workspace = true, optional = true } diff --git a/crates/top/rerun/src/commands/entrypoint.rs b/crates/top/rerun/src/commands/entrypoint.rs index 7c240a42cb0a..db076598dfb8 100644 --- a/crates/top/rerun/src/commands/entrypoint.rs +++ b/crates/top/rerun/src/commands/entrypoint.rs @@ -596,7 +596,7 @@ where } } } else { - run_impl(main_thread_token, build_info, call_source, args) + run_in_tokio(main_thread_token, build_info, call_source, args) }; match res { @@ -618,6 +618,29 @@ where } } +/// Ensures that we are running in the context of a tokio runtime. +fn run_in_tokio( + main_thread_token: crate::MainThreadToken, + build_info: re_build_info::BuildInfo, + call_source: CallSource, + args: Args, +) -> anyhow::Result<()> { + // tokio is a hard dependency as of our gRPC migration, + // so we must ensure it is always available: + if let Ok(handle) = tokio::runtime::Handle::try_current() { + // This thread already has a tokio runtime. + let _guard = handle.enter(); + run_impl(main_thread_token, build_info, call_source, args) + } else { + // We don't have a runtime yet, create one now. + let mut builder = tokio::runtime::Builder::new_multi_thread(); + builder.enable_all(); + let rt = builder.build()?; + let _guard = rt.enter(); + run_impl(main_thread_token, build_info, call_source, args) + } +} + fn run_impl( _main_thread_token: crate::MainThreadToken, _build_info: re_build_info::BuildInfo, From a03cf18992b98d8e017f40d22e05c46c9a23ac50 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Mon, 27 Jan 2025 16:32:36 +0100 Subject: [PATCH 07/87] Expose `grpc` variants for all `connect` and `spawn` in Python SDK --- crates/top/re_sdk/src/log_sink.rs | 4 +- crates/top/re_sdk/src/recording_stream.rs | 79 +++++++++++++++++++++++ crates/top/re_sdk/src/spawn.rs | 2 +- rerun_py/rerun_sdk/rerun/blueprint/api.py | 70 +++++++++++++++++++- rerun_py/src/python_bridge.rs | 65 +++++++++++++++++-- 5 files changed, 211 insertions(+), 9 deletions(-) diff --git a/crates/top/re_sdk/src/log_sink.rs b/crates/top/re_sdk/src/log_sink.rs index 048408b74fdc..7e61c7628c43 100644 --- a/crates/top/re_sdk/src/log_sink.rs +++ b/crates/top/re_sdk/src/log_sink.rs @@ -383,9 +383,9 @@ impl GrpcSink { /// GrpcSink::new("http://127.0.0.1:9434"); /// ``` #[inline] - pub fn new(addr: impl Into) -> Self { + pub fn new(url: impl Into) -> Self { Self { - client: MessageProxyClient::new(addr, Default::default()), + client: MessageProxyClient::new(url, Default::default()), } } } diff --git a/crates/top/re_sdk/src/recording_stream.rs b/crates/top/re_sdk/src/recording_stream.rs index fef81b467d02..ac63c81672bc 100644 --- a/crates/top/re_sdk/src/recording_stream.rs +++ b/crates/top/re_sdk/src/recording_stream.rs @@ -1838,6 +1838,38 @@ impl RecordingStream { self.set_sink(Box::new(sink)); } + /// Swaps the underlying sink for a [`crate::log_sink::GrpcSink`] sink pre-configured to use + /// the specified address. + /// + /// See also [`Self::connect_opts`] if you wish to configure the connection. + /// + /// This is a convenience wrapper for [`Self::set_sink`] that upholds the same guarantees in + /// terms of data durability and ordering. + /// See [`Self::set_sink`] for more information. + pub fn connect_grpc(&self) { + self.connect_grpc_opts(format!( + "http://127.0.0.1:{}", + re_grpc_server::DEFAULT_SERVER_PORT + )); + } + + /// Swaps the underlying sink for a [`crate::log_sink::GrpcSink`] sink pre-configured to use + /// the specified address. + /// + /// This is a convenience wrapper for [`Self::set_sink`] that upholds the same guarantees in + /// terms of data durability and ordering. + /// See [`Self::set_sink`] for more information. + pub fn connect_grpc_opts(&self, url: impl Into) { + if forced_sink_path().is_some() { + re_log::debug!("Ignored setting new GrpcSink since {ENV_FORCE_SAVE} is set"); + return; + } + + let sink = crate::log_sink::GrpcSink::new(url); + + self.set_sink(Box::new(sink)); + } + /// Spawns a new Rerun Viewer process from an executable available in PATH, then swaps the /// underlying sink for a [`crate::log_sink::TcpSink`] sink pre-configured to send data to that /// new process. @@ -1893,6 +1925,53 @@ impl RecordingStream { Ok(()) } + /// Spawns a new Rerun Viewer process from an executable available in PATH, then swaps the + /// underlying sink for a [`crate::log_sink::GrpcSink`] sink pre-configured to send data to that + /// new process. + /// + /// If a Rerun Viewer is already listening on this port, the stream will be redirected to + /// that viewer instead of starting a new one. + /// + /// See also [`Self::spawn_grpc_opts`] if you wish to configure the behavior of thew Rerun process + /// as well as the underlying connection. + /// + /// This is a convenience wrapper for [`Self::set_sink`] that upholds the same guarantees in + /// terms of data durability and ordering. + /// See [`Self::set_sink`] for more information. + pub fn spawn_grpc(&self) -> RecordingStreamResult<()> { + self.spawn_grpc_opts(&Default::default()) + } + + /// Spawns a new Rerun Viewer process from an executable available in PATH, then swaps the + /// underlying sink for a [`crate::log_sink::GrpcSink`] sink pre-configured to send data to that + /// new process. + /// + /// If a Rerun Viewer is already listening on this port, the stream will be redirected to + /// that viewer instead of starting a new one. + /// + /// The behavior of the spawned Viewer can be configured via `opts`. + /// If you're fine with the default behavior, refer to the simpler [`Self::spawn`]. + /// + /// This is a convenience wrapper for [`Self::set_sink`] that upholds the same guarantees in + /// terms of data durability and ordering. + /// See [`Self::set_sink`] for more information. + pub fn spawn_grpc_opts(&self, opts: &crate::SpawnOptions) -> RecordingStreamResult<()> { + if !self.is_enabled() { + re_log::debug!("Rerun disabled - call to spawn() ignored"); + return Ok(()); + } + if forced_sink_path().is_some() { + re_log::debug!("Ignored setting new TcpSink since {ENV_FORCE_SAVE} is set"); + return Ok(()); + } + + crate::spawn(opts)?; + + self.connect_grpc_opts(format!("http://127.0.0.1:{}", opts.connect_addr())); + + Ok(()) + } + /// Swaps the underlying sink for a [`crate::sink::MemorySink`] sink and returns the associated /// [`MemorySinkStorage`]. /// diff --git a/crates/top/re_sdk/src/spawn.rs b/crates/top/re_sdk/src/spawn.rs index 8a243ff37fc1..2ab2dbec39e3 100644 --- a/crates/top/re_sdk/src/spawn.rs +++ b/crates/top/re_sdk/src/spawn.rs @@ -140,7 +140,7 @@ impl std::fmt::Debug for SpawnError { } } -/// Spawns a new Rerun Viewer process ready to listen for TCP connections. +/// Spawns a new Rerun Viewer process ready to listen for connections. /// /// If there is already a process listening on this port (Rerun or not), this function returns `Ok` /// WITHOUT spawning a `rerun` process (!). diff --git a/rerun_py/rerun_sdk/rerun/blueprint/api.py b/rerun_py/rerun_sdk/rerun/blueprint/api.py index c40b442ef1d2..4a9af2f10e21 100644 --- a/rerun_py/rerun_sdk/rerun/blueprint/api.py +++ b/rerun_py/rerun_sdk/rerun/blueprint/api.py @@ -600,7 +600,50 @@ def connect( blueprint_stream.set_time_sequence("blueprint", 0) # type: ignore[attr-defined] self._log_to_stream(blueprint_stream) - bindings.connect_blueprint(addr, make_active, make_default, blueprint_stream.to_native()) + bindings.connect_tcp_blueprint(addr, make_active, make_default, blueprint_stream.to_native()) + + def connect_grpc( + self, + application_id: str, + *, + url: str | None = None, + make_active: bool = True, + make_default: bool = True, + ) -> None: + """ + Connect to a remote Rerun Viewer on the given HTTP(S) URL and send this blueprint. + + Parameters + ---------- + application_id: + The application ID to use for this blueprint. This must match the application ID used + when initiating rerun for any data logging you wish to associate with this blueprint. + url: + The HTTP(S) URL to connect to + make_active: + Immediately make this the active blueprint for the associated `app_id`. + Note that setting this to `false` does not mean the blueprint may not still end + up becoming active. In particular, if `make_default` is true and there is no other + currently active blueprint. + make_default: + Make this the default blueprint for the `app_id`. + The default blueprint will be used as the template when the user resets the + blueprint for the app. It will also become the active blueprint if no other + blueprint is currently active. + + """ + blueprint_stream = RecordingStream( + bindings.new_blueprint( + application_id=application_id, + make_default=False, + make_thread_default=False, + default_enabled=True, + ) + ) + blueprint_stream.set_time_sequence("blueprint", 0) # type: ignore[attr-defined] + self._log_to_stream(blueprint_stream) + + bindings.connect_grpc_blueprint(url, make_active, make_default, blueprint_stream.to_native()) def save(self, application_id: str, path: str | None = None) -> None: """ @@ -656,6 +699,31 @@ def spawn( _spawn_viewer(port=port, memory_limit=memory_limit, hide_welcome_screen=hide_welcome_screen) self.connect(application_id=application_id, addr=f"127.0.0.1:{port}") + def spawn_grpc( + self, application_id: str, port: int = 9876, memory_limit: str = "75%", hide_welcome_screen: bool = False + ) -> None: + """ + Spawn a Rerun viewer with this blueprint. + + Parameters + ---------- + application_id: + The application ID to use for this blueprint. This must match the application ID used + when initiating rerun for any data logging you wish to associate with this blueprint. + port: + The port to listen on. + memory_limit: + An upper limit on how much memory the Rerun Viewer should use. + When this limit is reached, Rerun will drop the oldest data. + Example: `16GB` or `50%` (of system total). + hide_welcome_screen: + Hide the normal Rerun welcome screen. + + """ + _spawn_viewer(port=port, memory_limit=memory_limit, hide_welcome_screen=hide_welcome_screen) + self.connect_grpc(application_id=application_id, url=f"http://127.0.0.1:{port}") + + BlueprintLike = Union[Blueprint, View, Container] """ diff --git a/rerun_py/src/python_bridge.rs b/rerun_py/src/python_bridge.rs index 9c3254e6f49b..3cc7f5a14987 100644 --- a/rerun_py/src/python_bridge.rs +++ b/rerun_py/src/python_bridge.rs @@ -139,8 +139,8 @@ fn rerun_bindings(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(binary_stream, m)?)?; m.add_function(wrap_pyfunction!(connect_tcp, m)?)?; m.add_function(wrap_pyfunction!(connect_tcp_blueprint, m)?)?; - #[cfg(feature = "remote")] m.add_function(wrap_pyfunction!(connect_grpc, m)?)?; + m.add_function(wrap_pyfunction!(connect_grpc_blueprint, m)?)?; m.add_function(wrap_pyfunction!(save, m)?)?; m.add_function(wrap_pyfunction!(save_blueprint, m)?)?; m.add_function(wrap_pyfunction!(stdout, m)?)?; @@ -675,16 +675,32 @@ fn connect_tcp_blueprint( } } -#[cfg(feature = "remote")] #[pyfunction] -#[pyo3(signature = (addr, recording = None))] -fn connect_grpc(addr: String, recording: Option<&PyRecordingStream>, py: Python<'_>) { +#[pyo3(signature = (url, default_blueprint = None, recording = None))] +fn connect_grpc( + url: Option, + default_blueprint: Option<&PyMemorySinkStorage>, + recording: Option<&PyRecordingStream>, + py: Python<'_>, +) { let Some(recording) = get_data_recording(recording) else { return; }; + use re_sdk::external::re_grpc_server::DEFAULT_SERVER_PORT; + let url = url.unwrap_or_else(|| format!("http://127.0.0.1:{DEFAULT_SERVER_PORT}")); + + if re_sdk::forced_sink_path().is_some() { + re_log::debug!("Ignored call to `connect()` since _RERUN_TEST_FORCE_SAVE is set"); + return; + } + py.allow_threads(|| { - let sink = re_sdk::sink::GrpcSink::new(addr); + let sink = re_sdk::sink::GrpcSink::new(url); + + if let Some(default_blueprint) = default_blueprint { + send_mem_sink_as_default_blueprint(&sink, default_blueprint); + } recording.set_sink(Box::new(sink)); @@ -692,6 +708,45 @@ fn connect_grpc(addr: String, recording: Option<&PyRecordingStream>, py: Python< }); } +#[pyfunction] +#[pyo3(signature = (url, make_active, make_default, blueprint_stream))] +/// Special binding for directly sending a blueprint stream to a connection. +fn connect_grpc_blueprint( + url: Option, + make_active: bool, + make_default: bool, + blueprint_stream: &PyRecordingStream, + py: Python<'_>, +) -> PyResult<()> { + use re_sdk::external::re_grpc_server::DEFAULT_SERVER_PORT; + let url = url.unwrap_or_else(|| format!("http://127.0.0.1:{DEFAULT_SERVER_PORT}")); + + if let Some(blueprint_id) = (*blueprint_stream).store_info().map(|info| info.store_id) { + // The call to save, needs to flush. + // Release the GIL in case any flushing behavior needs to cleanup a python object. + py.allow_threads(|| { + // Flush all the pending blueprint messages before we include the Ready message + blueprint_stream.flush_blocking(); + + let activation_cmd = BlueprintActivationCommand { + blueprint_id, + make_active, + make_default, + }; + + blueprint_stream.record_msg(activation_cmd.into()); + + blueprint_stream.connect_grpc_opts(url); + flush_garbage_queue(); + }); + Ok(()) + } else { + Err(PyRuntimeError::new_err( + "Blueprint stream has no store info".to_owned(), + )) + } +} + #[pyfunction] #[pyo3(signature = (path, default_blueprint = None, recording = None))] fn save( From 894cc4fc947a5e53a53c7ed3e21f5a84040ac648 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Mon, 27 Jan 2025 16:38:53 +0100 Subject: [PATCH 08/87] Expose gRPC `spawn`/`connect` in rust `RecordingStream` --- crates/top/re_sdk/src/recording_stream.rs | 104 +++++++++++++++++++++- 1 file changed, 103 insertions(+), 1 deletion(-) diff --git a/crates/top/re_sdk/src/recording_stream.rs b/crates/top/re_sdk/src/recording_stream.rs index ac63c81672bc..c046f6aa49e5 100644 --- a/crates/top/re_sdk/src/recording_stream.rs +++ b/crates/top/re_sdk/src/recording_stream.rs @@ -385,6 +385,51 @@ impl RecordingStreamBuilder { } } + /// Creates a new [`RecordingStream`] that is pre-configured to stream the data through to a + /// remote Rerun instance. + /// + /// See also [`Self::connect_opts`] if you wish to configure the connection. + /// + /// ## Example + /// + /// ```no_run + /// let rec = re_sdk::RecordingStreamBuilder::new("rerun_example_app").connect_grpc()?; + /// # Ok::<(), Box>(()) + /// ``` + pub fn connect_grpc(self) -> RecordingStreamResult { + self.connect_grpc_opts(format!( + "http://127.0.0.1:{}", + re_grpc_server::DEFAULT_SERVER_PORT + )) + } + + /// Creates a new [`RecordingStream`] that is pre-configured to stream the data through to a + /// remote Rerun instance. + /// + /// ## Example + /// + /// ```no_run + /// let rec = re_sdk::RecordingStreamBuilder::new("rerun_example_app") + /// .connect_grpc_opts("http://127.0.0.1:1852")?; + /// # Ok::<(), Box>(()) + /// ``` + pub fn connect_grpc_opts( + self, + url: impl Into, + ) -> RecordingStreamResult { + let (enabled, store_info, batcher_config) = self.into_args(); + if enabled { + RecordingStream::new( + store_info, + batcher_config, + Box::new(crate::log_sink::GrpcSink::new(url)), + ) + } else { + re_log::debug!("Rerun disabled - call to connect() ignored"); + Ok(RecordingStream::disabled()) + } + } + /// Creates a new [`RecordingStream`] that is pre-configured to stream the data through to an /// RRD file on disk. /// @@ -511,6 +556,63 @@ impl RecordingStreamBuilder { self.connect_tcp_opts(connect_addr, flush_timeout) } + /// Spawns a new Rerun Viewer process from an executable available in PATH, then creates a new + /// [`RecordingStream`] that is pre-configured to stream the data through to that viewer over TCP. + /// + /// If a Rerun Viewer is already listening on this port, the stream will be redirected to + /// that viewer instead of starting a new one. + /// + /// See also [`Self::spawn_grpc_opts`] if you wish to configure the behavior of thew Rerun process + /// as well as the underlying connection. + /// + /// ## Example + /// + /// ```no_run + /// let rec = re_sdk::RecordingStreamBuilder::new("rerun_example_app").spawn_grpc()?; + /// # Ok::<(), Box>(()) + /// ``` + pub fn spawn_grpc(self) -> RecordingStreamResult { + self.spawn_grpc_opts(&Default::default()) + } + + /// Spawns a new Rerun Viewer process from an executable available in PATH, then creates a new + /// [`RecordingStream`] that is pre-configured to stream the data through to that viewer over TCP. + /// + /// If a Rerun Viewer is already listening on this port, the stream will be redirected to + /// that viewer instead of starting a new one. + /// + /// The behavior of the spawned Viewer can be configured via `opts`. + /// If you're fine with the default behavior, refer to the simpler [`Self::spawn_grpc`]. + /// + /// ## Example + /// + /// ```no_run + /// let rec = re_sdk::RecordingStreamBuilder::new("rerun_example_app") + /// .spawn_grpc_opts(&re_sdk::SpawnOptions::default())?; + /// # Ok::<(), Box>(()) + /// ``` + pub fn spawn_grpc_opts( + self, + opts: &crate::SpawnOptions, + ) -> RecordingStreamResult { + if !self.is_enabled() { + re_log::debug!("Rerun disabled - call to spawn() ignored"); + return Ok(RecordingStream::disabled()); + } + + let url = format!("http://{}", opts.connect_addr()); + + // NOTE: If `_RERUN_TEST_FORCE_SAVE` is set, all recording streams will write to disk no matter + // what, thus spawning a viewer is pointless (and probably not intended). + if forced_sink_path().is_some() { + return self.connect_grpc_opts(url); + } + + crate::spawn(opts)?; + + self.connect_grpc_opts(url) + } + /// Creates a new [`RecordingStream`] that is pre-configured to stream the data through to a /// web-based Rerun viewer via WebSockets. /// @@ -1967,7 +2069,7 @@ impl RecordingStream { crate::spawn(opts)?; - self.connect_grpc_opts(format!("http://127.0.0.1:{}", opts.connect_addr())); + self.connect_grpc_opts(format!("http://{}", opts.connect_addr())); Ok(()) } From c4257b33ce4e4171d502e30bbdf76a81b268eefa Mon Sep 17 00:00:00 2001 From: jprochazk Date: Mon, 27 Jan 2025 16:47:02 +0100 Subject: [PATCH 09/87] Expose gRPC `connect`/`spawn` in C/C++ --- crates/top/rerun_c/src/lib.rs | 59 ++++++++++++++++++++++++ rerun_cpp/src/rerun/recording_stream.cpp | 14 ++++++ rerun_cpp/src/rerun/recording_stream.hpp | 18 ++++++++ 3 files changed, 91 insertions(+) diff --git a/crates/top/rerun_c/src/lib.rs b/crates/top/rerun_c/src/lib.rs index 40f23396cbb8..264d75ff486a 100644 --- a/crates/top/rerun_c/src/lib.rs +++ b/crates/top/rerun_c/src/lib.rs @@ -596,6 +596,32 @@ pub extern "C" fn rr_recording_stream_connect( } } +#[allow(clippy::result_large_err)] +fn rr_recording_stream_connect_grpc_impl( + stream: CRecordingStream, + url: CStringView, +) -> Result<(), CError> { + let stream = recording_stream(stream)?; + + let url = url.as_str("url")?; + + stream.connect_grpc_opts(url); + + Ok(()) +} + +#[allow(unsafe_code)] +#[no_mangle] +pub extern "C" fn rr_recording_stream_connect_grpc( + id: CRecordingStream, + url: CStringView, + error: *mut CError, +) { + if let Err(err) = rr_recording_stream_connect_grpc_impl(id, url) { + err.write_error(error); + } +} + #[allow(clippy::result_large_err)] fn rr_recording_stream_spawn_impl( stream: CRecordingStream, @@ -636,6 +662,39 @@ pub extern "C" fn rr_recording_stream_spawn( } } +#[allow(clippy::result_large_err)] +fn rr_recording_stream_spawn_grpc_impl( + stream: CRecordingStream, + spawn_opts: *const CSpawnOptions, +) -> Result<(), CError> { + let stream = recording_stream(stream)?; + + let spawn_opts = if spawn_opts.is_null() { + re_sdk::SpawnOptions::default() + } else { + let spawn_opts = ptr::try_ptr_as_ref(spawn_opts, "spawn_opts")?; + spawn_opts.as_rust()? + }; + + stream + .spawn_grpc_opts(&spawn_opts) + .map_err(|err| CError::new(CErrorCode::RecordingStreamSpawnFailure, &err.to_string()))?; + + Ok(()) +} + +#[allow(unsafe_code)] +#[no_mangle] +pub extern "C" fn rr_recording_stream_spawn_grpc( + id: CRecordingStream, + spawn_opts: *const CSpawnOptions, + error: *mut CError, +) { + if let Err(err) = rr_recording_stream_spawn_grpc_impl(id, spawn_opts) { + err.write_error(error); + } +} + #[allow(clippy::result_large_err)] fn rr_recording_stream_save_impl( stream: CRecordingStream, diff --git a/rerun_cpp/src/rerun/recording_stream.cpp b/rerun_cpp/src/rerun/recording_stream.cpp index e645425460eb..0e04aba79462 100644 --- a/rerun_cpp/src/rerun/recording_stream.cpp +++ b/rerun_cpp/src/rerun/recording_stream.cpp @@ -119,6 +119,12 @@ namespace rerun { return status; } + Error RecordingStream::connect_grpc(std::string_view url) const { + rr_error status = {}; + rr_recording_stream_connect_grpc(_id, detail::to_rr_string(url), &status); + return status; + } + Error RecordingStream::spawn(const SpawnOptions& options, float flush_timeout_sec) const { rr_spawn_options rerun_c_options = {}; options.fill_rerun_c_struct(rerun_c_options); @@ -127,6 +133,14 @@ namespace rerun { return status; } + Error RecordingStream::spawn_grpc(const SpawnOptions& options) const { + rr_spawn_options rerun_c_options = {}; + options.fill_rerun_c_struct(rerun_c_options); + rr_error status = {}; + rr_recording_stream_spawn_grpc(_id, &rerun_c_options, &status); + return status; + } + Error RecordingStream::save(std::string_view path) const { rr_error status = {}; rr_recording_stream_save(_id, detail::to_rr_string(path), &status); diff --git a/rerun_cpp/src/rerun/recording_stream.hpp b/rerun_cpp/src/rerun/recording_stream.hpp index 88f68415373d..9ea2ff280a62 100644 --- a/rerun_cpp/src/rerun/recording_stream.hpp +++ b/rerun_cpp/src/rerun/recording_stream.hpp @@ -161,6 +161,13 @@ namespace rerun { std::string_view tcp_addr = "127.0.0.1:9876", float flush_timeout_sec = 2.0 ) const; + /// Connect to a remote Rerun Viewer on the given HTTP(S) URL. + /// + /// Requires that you first start a Rerun Viewer by typing 'rerun' in a terminal. + /// + /// This function returns immediately. + Error connect_grpc(std::string_view url = "http://127.0.0.1:1852") const; + /// Spawns a new Rerun Viewer process from an executable available in PATH, then connects to it /// over TCP. /// @@ -177,6 +184,17 @@ namespace rerun { /// timeout, and can cause a call to `flush` to block indefinitely. Error spawn(const SpawnOptions& options = {}, float flush_timeout_sec = 2.0) const; + /// Spawns a new Rerun Viewer process from an executable available in PATH, then connects to it + /// over gRPC. + /// + /// If a Rerun Viewer is already listening on this port, the stream will be redirected to + /// that viewer instead of starting a new one. + /// + /// ## Parameters + /// options: + /// See `rerun::SpawnOptions` for more information. + Error spawn_grpc(const SpawnOptions& options = {}) const; + /// @see RecordingStream::spawn template Error spawn( From 8bd2f096e1e3b01eac16872af8c8fd7c4e958405 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Mon, 27 Jan 2025 16:55:03 +0100 Subject: [PATCH 10/87] Allow setting IP when serving grpc --- crates/store/re_grpc_server/src/lib.rs | 34 +++++++++++++++---------- crates/store/re_grpc_server/src/main.rs | 4 +-- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/crates/store/re_grpc_server/src/lib.rs b/crates/store/re_grpc_server/src/lib.rs index 1d362da814ba..4450bd95def0 100644 --- a/crates/store/re_grpc_server/src/lib.rs +++ b/crates/store/re_grpc_server/src/lib.rs @@ -1,9 +1,8 @@ //! Server implementation of an in-memory Storage Node. use std::collections::VecDeque; +use std::net::IpAddr; use std::net::Ipv4Addr; -use std::net::SocketAddr; -use std::net::SocketAddrV4; use std::pin::Pin; use re_byte_size::SizeBytes; @@ -23,22 +22,28 @@ use tonic::transport::server::TcpIncoming; use tonic::transport::Server; pub const DEFAULT_SERVER_PORT: u16 = 1852; +pub const DEFAULT_SERVER_IP: IpAddr = IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)); pub const DEFAULT_MEMORY_LIMIT: MemoryLimit = MemoryLimit::UNLIMITED; /// Listen for incoming clients on `addr`. /// /// The server runs on the current task. -pub async fn serve(port: u16, memory_limit: MemoryLimit) -> Result<(), tonic::transport::Error> { - serve_impl(port, MessageProxy::new(memory_limit)).await +pub async fn serve( + ip: IpAddr, + port: u16, + memory_limit: MemoryLimit, +) -> Result<(), tonic::transport::Error> { + serve_impl(ip, port, MessageProxy::new(memory_limit)).await } -async fn serve_impl(port: u16, message_proxy: MessageProxy) -> Result<(), tonic::transport::Error> { - let tcp_listener = TcpListener::bind(SocketAddr::V4(SocketAddrV4::new( - Ipv4Addr::new(0, 0, 0, 0), - port, - ))) - .await - .unwrap_or_else(|err| panic!("failed to bind listener on port {port}: {err}")); +async fn serve_impl( + ip: IpAddr, + port: u16, + message_proxy: MessageProxy, +) -> Result<(), tonic::transport::Error> { + let tcp_listener = TcpListener::bind((ip, port)) + .await + .unwrap_or_else(|err| panic!("failed to bind listener on port {port}: {err}")); let incoming = TcpIncoming::from_listener(tcp_listener, true, None).expect("failed to init listener"); @@ -71,19 +76,20 @@ async fn serve_impl(port: u16, message_proxy: MessageProxy) -> Result<(), tonic: } pub fn spawn_with_recv( + ip: IpAddr, port: u16, memory_limit: MemoryLimit, ) -> re_smart_channel::Receiver { let (channel_tx, channel_rx) = re_smart_channel::smart_channel( re_smart_channel::SmartMessageSource::MessageProxy { - url: format!("http://localhost:{port}"), + url: format!("http://127.0.0.1:{port}"), }, re_smart_channel::SmartChannelSource::MessageProxy { - url: format!("http://localhost:{port}"), + url: format!("http://127.0.0.1:{port}"), }, ); let (message_proxy, mut broadcast_rx) = MessageProxy::new_with_recv(memory_limit); - tokio::spawn(serve_impl(port, message_proxy)); + tokio::spawn(serve_impl(ip, port, message_proxy)); tokio::spawn(async move { loop { let msg = match broadcast_rx.recv().await { diff --git a/crates/store/re_grpc_server/src/main.rs b/crates/store/re_grpc_server/src/main.rs index 58dfe287b02f..a042caea327b 100644 --- a/crates/store/re_grpc_server/src/main.rs +++ b/crates/store/re_grpc_server/src/main.rs @@ -1,8 +1,8 @@ -use re_grpc_server::{serve, DEFAULT_MEMORY_LIMIT, DEFAULT_SERVER_PORT}; +use re_grpc_server::{serve, DEFAULT_MEMORY_LIMIT, DEFAULT_SERVER_IP, DEFAULT_SERVER_PORT}; #[tokio::main(flavor = "current_thread")] async fn main() -> Result<(), tonic::transport::Error> { re_log::setup_logging(); - serve(DEFAULT_SERVER_PORT, DEFAULT_MEMORY_LIMIT).await + serve(DEFAULT_SERVER_IP, DEFAULT_SERVER_PORT, DEFAULT_MEMORY_LIMIT).await } From c33e8efdb4051d675909855204b786199fda51e1 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Mon, 27 Jan 2025 17:19:35 +0100 Subject: [PATCH 11/87] Host gRPC server by default --- crates/top/rerun/src/commands/entrypoint.rs | 23 +++++++------------ crates/utils/re_smart_channel/src/lib.rs | 5 ++-- crates/viewer/re_viewer/src/app.rs | 7 +++--- crates/viewer/re_viewer/src/app_state.rs | 2 +- .../re_viewer/src/ui/recordings_panel.rs | 8 ++++--- crates/viewer/re_viewer/src/ui/top_panel.rs | 5 ++-- 6 files changed, 21 insertions(+), 29 deletions(-) diff --git a/crates/top/rerun/src/commands/entrypoint.rs b/crates/top/rerun/src/commands/entrypoint.rs index db076598dfb8..441658cf6f52 100644 --- a/crates/top/rerun/src/commands/entrypoint.rs +++ b/crates/top/rerun/src/commands/entrypoint.rs @@ -738,13 +738,14 @@ fn run_impl( ); is_another_viewer_running = true; } else { - let server_options = re_sdk_comms::ServerOptions { - max_latency_sec: parse_max_latency(args.drop_at_latency.as_ref()), - quiet: false, - }; - let tcp_listener: Receiver = - re_sdk_comms::serve(&args.bind, args.port, server_options)?; - rxs.push(tcp_listener); + let server_memory_limit = re_memory::MemoryLimit::parse(&args.server_memory_limit) + .map_err(|err| anyhow::format_err!("Bad --server-memory-limit: {err}"))?; + let server: Receiver = re_grpc_server::spawn_with_recv( + args.bind.parse()?, + args.port, + server_memory_limit, + ); + rxs.push(server); } } @@ -1007,14 +1008,6 @@ fn parse_size(size: &str) -> anyhow::Result<[f32; 2]> { .ok_or_else(|| anyhow::anyhow!("Invalid size {:?}, expected e.g. 800x600", size)) } -#[cfg(feature = "server")] -fn parse_max_latency(max_latency: Option<&String>) -> f32 { - max_latency.as_ref().map_or(f32::INFINITY, |time| { - re_format::parse_duration(time) - .unwrap_or_else(|err| panic!("Failed to parse max_latency ({max_latency:?}): {err}")) - }) -} - // --- io --- // TODO(cmc): dedicated module for io utils, especially stdio streaming in and out. diff --git a/crates/utils/re_smart_channel/src/lib.rs b/crates/utils/re_smart_channel/src/lib.rs index 33eeeea11064..f678a7719239 100644 --- a/crates/utils/re_smart_channel/src/lib.rs +++ b/crates/utils/re_smart_channel/src/lib.rs @@ -81,14 +81,13 @@ impl std::fmt::Display for SmartChannelSource { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Self::File(path) => path.display().fmt(f), - Self::RrdHttpStream { url, follow: _ } - | Self::RerunGrpcStream { url } - | Self::MessageProxy { url } => url.fmt(f), + Self::RrdHttpStream { url, follow: _ } | Self::RerunGrpcStream { url } => url.fmt(f), Self::RrdWebEventListener => "Web event listener".fmt(f), Self::JsChannel { channel_name } => write!(f, "Javascript channel: {channel_name}"), Self::Sdk => "SDK".fmt(f), Self::WsClient { ws_server_url } => ws_server_url.fmt(f), Self::TcpServer { port } => write!(f, "TCP server, port {port}"), + Self::MessageProxy { url } => write!(f, "gRPC server: {url}"), Self::Stdin => "Standard input".fmt(f), } } diff --git a/crates/viewer/re_viewer/src/app.rs b/crates/viewer/re_viewer/src/app.rs index e8b8013ae8f8..8392167e2bee 100644 --- a/crates/viewer/re_viewer/src/app.rs +++ b/crates/viewer/re_viewer/src/app.rs @@ -539,13 +539,13 @@ impl App { self.rx.retain(|r| match r.source() { SmartChannelSource::File(_) | SmartChannelSource::RrdHttpStream { .. } - | SmartChannelSource::RerunGrpcStream { .. } - | SmartChannelSource::MessageProxy { .. } => false, + | SmartChannelSource::RerunGrpcStream { .. } => false, SmartChannelSource::WsClient { .. } | SmartChannelSource::JsChannel { .. } | SmartChannelSource::RrdWebEventListener | SmartChannelSource::Sdk + | SmartChannelSource::MessageProxy { .. } | SmartChannelSource::TcpServer { .. } | SmartChannelSource::Stdin => true, }); @@ -1575,7 +1575,6 @@ impl App { SmartChannelSource::File(_) | SmartChannelSource::RrdHttpStream { .. } | SmartChannelSource::RerunGrpcStream { .. } - | SmartChannelSource::MessageProxy { .. } | SmartChannelSource::Stdin | SmartChannelSource::RrdWebEventListener | SmartChannelSource::Sdk @@ -1584,7 +1583,7 @@ impl App { return true; // We expect data soon, so fade-in } - SmartChannelSource::TcpServer { .. } => { + SmartChannelSource::TcpServer { .. } | SmartChannelSource::MessageProxy { .. } => { // We start a TCP server by default in native rerun, i.e. when just running `rerun`, // and in that case fading in the welcome screen would be slightly annoying. // However, we also use the TCP server for sending data from the logging SDKs diff --git a/crates/viewer/re_viewer/src/app_state.rs b/crates/viewer/re_viewer/src/app_state.rs index 43e21e465632..dde57820d4cb 100644 --- a/crates/viewer/re_viewer/src/app_state.rs +++ b/crates/viewer/re_viewer/src/app_state.rs @@ -643,7 +643,6 @@ fn recording_config_entry<'cfgs>( re_smart_channel::SmartChannelSource::File(_) | re_smart_channel::SmartChannelSource::RrdHttpStream { follow: false, .. } | re_smart_channel::SmartChannelSource::RerunGrpcStream { .. } - | re_smart_channel::SmartChannelSource::MessageProxy { .. } | re_smart_channel::SmartChannelSource::RrdWebEventListener => PlayState::Playing, // Live data - follow it! @@ -651,6 +650,7 @@ fn recording_config_entry<'cfgs>( | re_smart_channel::SmartChannelSource::Sdk | re_smart_channel::SmartChannelSource::WsClient { .. } | re_smart_channel::SmartChannelSource::TcpServer { .. } + | re_smart_channel::SmartChannelSource::MessageProxy { .. } | re_smart_channel::SmartChannelSource::Stdin | re_smart_channel::SmartChannelSource::JsChannel { .. } => PlayState::Following, } diff --git a/crates/viewer/re_viewer/src/ui/recordings_panel.rs b/crates/viewer/re_viewer/src/ui/recordings_panel.rs index 6ce6a3f08d82..af6528560615 100644 --- a/crates/viewer/re_viewer/src/ui/recordings_panel.rs +++ b/crates/viewer/re_viewer/src/ui/recordings_panel.rs @@ -60,14 +60,14 @@ fn loading_receivers_ui(ctx: &ViewerContext<'_>, rx: &ReceiveSet, ui: &m // We only show things we know are very-soon-to-be recordings: SmartChannelSource::File(path) => format!("Loading {}…", path.display()), SmartChannelSource::RrdHttpStream { url, .. } - | SmartChannelSource::RerunGrpcStream { url } - | SmartChannelSource::MessageProxy { url } => format!("Loading {url}…"), + | SmartChannelSource::RerunGrpcStream { url } => format!("Loading {url}…"), SmartChannelSource::RrdWebEventListener | SmartChannelSource::JsChannel { .. } | SmartChannelSource::Sdk | SmartChannelSource::WsClient { .. } | SmartChannelSource::TcpServer { .. } + | re_smart_channel::SmartChannelSource::MessageProxy { .. } | SmartChannelSource::Stdin => { // These show up in the top panel - see `top_panel.rs`. continue; @@ -92,7 +92,9 @@ fn loading_receivers_ui(ctx: &ViewerContext<'_>, rx: &ReceiveSet, ui: &m resp }), ); - if let SmartChannelSource::TcpServer { .. } = source.as_ref() { + if let SmartChannelSource::TcpServer { .. } | SmartChannelSource::MessageProxy { .. } = + source.as_ref() + { response.on_hover_text("You can connect to this viewer from a Rerun SDK"); } } diff --git a/crates/viewer/re_viewer/src/ui/top_panel.rs b/crates/viewer/re_viewer/src/ui/top_panel.rs index 36883579ef93..e390b75528f1 100644 --- a/crates/viewer/re_viewer/src/ui/top_panel.rs +++ b/crates/viewer/re_viewer/src/ui/top_panel.rs @@ -201,13 +201,12 @@ fn connection_status_ui(ui: &mut egui::Ui, rx: &ReceiveSet | SmartChannelSource::Stdin | SmartChannelSource::RrdHttpStream { .. } | SmartChannelSource::RerunGrpcStream { .. } - | SmartChannelSource::MessageProxy { .. } | SmartChannelSource::RrdWebEventListener | SmartChannelSource::JsChannel { .. } | SmartChannelSource::Sdk | SmartChannelSource::WsClient { .. } => None, - SmartChannelSource::TcpServer { .. } => { + SmartChannelSource::TcpServer { .. } | SmartChannelSource::MessageProxy { .. } => { Some("Waiting for an SDK to connect".to_owned()) } }; @@ -228,7 +227,7 @@ fn connection_status_ui(ui: &mut egui::Ui, rx: &ReceiveSet re_smart_channel::SmartChannelSource::RrdHttpStream { url, .. } | re_smart_channel::SmartChannelSource::RerunGrpcStream { url } | re_smart_channel::SmartChannelSource::MessageProxy { url } => { - format!("Loading {url}…") + format!("Waiting for data on {url}…") } re_smart_channel::SmartChannelSource::RrdWebEventListener | re_smart_channel::SmartChannelSource::JsChannel { .. } => { From 7c9e742d8f0c4be1b0905792ea52151d5e86f121 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Tue, 28 Jan 2025 10:06:07 +0100 Subject: [PATCH 12/87] Dedupe variable --- crates/top/rerun/src/commands/entrypoint.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/crates/top/rerun/src/commands/entrypoint.rs b/crates/top/rerun/src/commands/entrypoint.rs index 441658cf6f52..300f7fc07b80 100644 --- a/crates/top/rerun/src/commands/entrypoint.rs +++ b/crates/top/rerun/src/commands/entrypoint.rs @@ -691,6 +691,10 @@ fn run_impl( } }; + #[cfg(feature = "server")] + let server_memory_limit = re_memory::MemoryLimit::parse(&args.server_memory_limit) + .map_err(|err| anyhow::format_err!("Bad --server-memory-limit: {err}"))?; + // Where do we get the data from? let rxs: Vec> = { let data_sources = args @@ -738,8 +742,6 @@ fn run_impl( ); is_another_viewer_running = true; } else { - let server_memory_limit = re_memory::MemoryLimit::parse(&args.server_memory_limit) - .map_err(|err| anyhow::format_err!("Bad --server-memory-limit: {err}"))?; let server: Receiver = re_grpc_server::spawn_with_recv( args.bind.parse()?, args.port, @@ -788,9 +790,6 @@ fn run_impl( #[cfg(feature = "server")] { - let server_memory_limit = re_memory::MemoryLimit::parse(&args.server_memory_limit) - .map_err(|err| anyhow::format_err!("Bad --server-memory-limit: {err}"))?; - // This is the server which the web viewer will talk to: let _ws_server = re_ws_comms::RerunServer::new( ReceiveSet::new(rxs), From 2a23554c7433dad74fef55d95b0f2b3322a83b44 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Tue, 28 Jan 2025 10:52:36 +0100 Subject: [PATCH 13/87] Change error message --- crates/store/re_grpc_client/src/message_proxy/write.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/store/re_grpc_client/src/message_proxy/write.rs b/crates/store/re_grpc_client/src/message_proxy/write.rs index e7bbafa28abe..512c41716b0a 100644 --- a/crates/store/re_grpc_client/src/message_proxy/write.rs +++ b/crates/store/re_grpc_client/src/message_proxy/write.rs @@ -113,7 +113,7 @@ async fn message_proxy_client( let endpoint = match Endpoint::from_shared(url) { Ok(endpoint) => endpoint, Err(err) => { - re_log::error!("Failed to connect to message proxy server: {err}"); + re_log::error!("Invalid message proxy server endpoint: {err}"); return; } }; From 95c9848fbd3ec7bc48fa6e36402b6381d4ae3ddb Mon Sep 17 00:00:00 2001 From: jprochazk Date: Tue, 28 Jan 2025 10:53:10 +0100 Subject: [PATCH 14/87] Refactor server entrypoint IP address parsing --- crates/store/re_grpc_server/src/lib.rs | 27 ++++++++----------- crates/store/re_grpc_server/src/main.rs | 15 +++++++++-- crates/top/rerun/src/commands/entrypoint.rs | 30 ++++++++++----------- 3 files changed, 39 insertions(+), 33 deletions(-) diff --git a/crates/store/re_grpc_server/src/lib.rs b/crates/store/re_grpc_server/src/lib.rs index 4450bd95def0..8c74704d7659 100644 --- a/crates/store/re_grpc_server/src/lib.rs +++ b/crates/store/re_grpc_server/src/lib.rs @@ -1,8 +1,7 @@ //! Server implementation of an in-memory Storage Node. use std::collections::VecDeque; -use std::net::IpAddr; -use std::net::Ipv4Addr; +use std::net::SocketAddr; use std::pin::Pin; use re_byte_size::SizeBytes; @@ -22,33 +21,30 @@ use tonic::transport::server::TcpIncoming; use tonic::transport::Server; pub const DEFAULT_SERVER_PORT: u16 = 1852; -pub const DEFAULT_SERVER_IP: IpAddr = IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)); pub const DEFAULT_MEMORY_LIMIT: MemoryLimit = MemoryLimit::UNLIMITED; /// Listen for incoming clients on `addr`. /// /// The server runs on the current task. pub async fn serve( - ip: IpAddr, - port: u16, + addr: SocketAddr, memory_limit: MemoryLimit, ) -> Result<(), tonic::transport::Error> { - serve_impl(ip, port, MessageProxy::new(memory_limit)).await + serve_impl(addr, MessageProxy::new(memory_limit)).await } async fn serve_impl( - ip: IpAddr, - port: u16, + addr: SocketAddr, message_proxy: MessageProxy, ) -> Result<(), tonic::transport::Error> { - let tcp_listener = TcpListener::bind((ip, port)) + let tcp_listener = TcpListener::bind(addr) .await - .unwrap_or_else(|err| panic!("failed to bind listener on port {port}: {err}")); + .unwrap_or_else(|err| panic!("failed to bind listener on {addr}: {err}")); let incoming = TcpIncoming::from_listener(tcp_listener, true, None).expect("failed to init listener"); - re_log::info!("Listening for gRPC connections on http://0.0.0.0:{port}"); + re_log::info!("Listening for gRPC connections on http://{addr}"); use tower_http::cors::{Any, CorsLayer}; let cors = CorsLayer::new() @@ -76,20 +72,19 @@ async fn serve_impl( } pub fn spawn_with_recv( - ip: IpAddr, - port: u16, + addr: SocketAddr, memory_limit: MemoryLimit, ) -> re_smart_channel::Receiver { let (channel_tx, channel_rx) = re_smart_channel::smart_channel( re_smart_channel::SmartMessageSource::MessageProxy { - url: format!("http://127.0.0.1:{port}"), + url: format!("http://{addr}"), }, re_smart_channel::SmartChannelSource::MessageProxy { - url: format!("http://127.0.0.1:{port}"), + url: format!("http://{addr}"), }, ); let (message_proxy, mut broadcast_rx) = MessageProxy::new_with_recv(memory_limit); - tokio::spawn(serve_impl(ip, port, message_proxy)); + tokio::spawn(serve_impl(addr, message_proxy)); tokio::spawn(async move { loop { let msg = match broadcast_rx.recv().await { diff --git a/crates/store/re_grpc_server/src/main.rs b/crates/store/re_grpc_server/src/main.rs index a042caea327b..4f5dd0dbdf38 100644 --- a/crates/store/re_grpc_server/src/main.rs +++ b/crates/store/re_grpc_server/src/main.rs @@ -1,8 +1,19 @@ -use re_grpc_server::{serve, DEFAULT_MEMORY_LIMIT, DEFAULT_SERVER_IP, DEFAULT_SERVER_PORT}; +use std::net::Ipv4Addr; +use std::net::SocketAddr; +use std::net::SocketAddrV4; + +use re_grpc_server::{serve, DEFAULT_MEMORY_LIMIT, DEFAULT_SERVER_PORT}; #[tokio::main(flavor = "current_thread")] async fn main() -> Result<(), tonic::transport::Error> { re_log::setup_logging(); - serve(DEFAULT_SERVER_IP, DEFAULT_SERVER_PORT, DEFAULT_MEMORY_LIMIT).await + serve( + SocketAddr::V4(SocketAddrV4::new( + Ipv4Addr::new(0, 0, 0, 0), + DEFAULT_SERVER_PORT, + )), + DEFAULT_MEMORY_LIMIT, + ) + .await } diff --git a/crates/top/rerun/src/commands/entrypoint.rs b/crates/top/rerun/src/commands/entrypoint.rs index 300f7fc07b80..e42b7b176a38 100644 --- a/crates/top/rerun/src/commands/entrypoint.rs +++ b/crates/top/rerun/src/commands/entrypoint.rs @@ -1,3 +1,5 @@ +use std::net::IpAddr; + use clap::{CommandFactory, Subcommand}; use itertools::Itertools; @@ -85,7 +87,7 @@ struct Args { /// What bind address IP to use. #[clap(long, default_value = "0.0.0.0")] - bind: String, + bind: IpAddr, /// Set a maximum input latency, e.g. "200ms" or "10s". /// @@ -691,6 +693,7 @@ fn run_impl( } }; + let server_addr = std::net::SocketAddr::new(args.bind, args.port); #[cfg(feature = "server")] let server_memory_limit = re_memory::MemoryLimit::parse(&args.server_memory_limit) .map_err(|err| anyhow::format_err!("Bad --server-memory-limit: {err}"))?; @@ -711,7 +714,7 @@ fn run_impl( // Instead of piping, just host a web-viewer that connects to the web-socket directly: WebViewerConfig { - bind_ip: args.bind, + bind_ip: args.bind.to_string(), web_port: args.web_viewer_port, source_url: Some(rerun_server_ws_url), force_wgpu_backend: args.renderer, @@ -734,19 +737,15 @@ fn run_impl( { // Check if there is already a viewer running and if so, send the data to it. use std::net::TcpStream; - let addr = std::net::SocketAddr::new(re_sdk::default_server_addr().ip(), args.port); - if TcpStream::connect_timeout(&addr, std::time::Duration::from_secs(1)).is_ok() { + if TcpStream::connect_timeout(&server_addr, std::time::Duration::from_secs(1)).is_ok() { re_log::info!( - %addr, + %server_addr, "A process is already listening at this address. Assuming it's a Rerun Viewer." ); is_another_viewer_running = true; } else { - let server: Receiver = re_grpc_server::spawn_with_recv( - args.bind.parse()?, - args.port, - server_memory_limit, - ); + let server: Receiver = + re_grpc_server::spawn_with_recv(server_addr, server_memory_limit); rxs.push(server); } } @@ -793,7 +792,7 @@ fn run_impl( // This is the server which the web viewer will talk to: let _ws_server = re_ws_comms::RerunServer::new( ReceiveSet::new(rxs), - &args.bind, + &args.bind.to_string(), args.ws_server_port, server_memory_limit, )?; @@ -807,7 +806,7 @@ fn run_impl( // This is the server that serves the Wasm+HTML: WebViewerConfig { - bind_ip: args.bind, + bind_ip: args.bind.to_string(), web_port: args.web_viewer_port, source_url: Some(_ws_server.server_url()), force_wgpu_backend: args.renderer, @@ -827,10 +826,11 @@ fn run_impl( return Ok(()); } } else if is_another_viewer_running { - let addr = std::net::SocketAddr::new(re_sdk::default_server_addr().ip(), args.port); - re_log::info!(%addr, "Another viewer is already running, streaming data to it."); + // Another viewer is already running on the specified address + let url = format!("http://{server_addr}"); + re_log::info!(%url, "Another viewer is already running, streaming data to it."); - let sink = re_sdk::sink::TcpSink::new(addr, re_sdk::default_flush_timeout()); + let sink = re_sdk::sink::GrpcSink::new(url); for rx in rxs { while rx.is_connected() { From 46563c493b2d96039831ab87168ffdd2f4a1ebd0 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Tue, 28 Jan 2025 10:53:16 +0100 Subject: [PATCH 15/87] Spawn with grpc by default --- rerun_py/rerun_sdk/rerun/__init__.py | 2 +- rerun_py/rerun_sdk/rerun/sinks.py | 89 +++++++++++++++++++++++++++- 2 files changed, 88 insertions(+), 3 deletions(-) diff --git a/rerun_py/rerun_sdk/rerun/__init__.py b/rerun_py/rerun_sdk/rerun/__init__.py index 94e514fa637b..0e6a3bcfadbb 100644 --- a/rerun_py/rerun_sdk/rerun/__init__.py +++ b/rerun_py/rerun_sdk/rerun/__init__.py @@ -352,7 +352,7 @@ def init( ) if spawn: - from rerun.sinks import spawn as _spawn + from rerun.sinks import spawn_grpc as _spawn _spawn(default_blueprint=default_blueprint) diff --git a/rerun_py/rerun_sdk/rerun/sinks.py b/rerun_py/rerun_sdk/rerun/sinks.py index 4ebd5d45bdb2..7c90aac6832f 100644 --- a/rerun_py/rerun_sdk/rerun/sinks.py +++ b/rerun_py/rerun_sdk/rerun/sinks.py @@ -139,10 +139,32 @@ def connect_tcp( def connect_grpc( - addr: str | None = None, + url: str | None = None, *, + default_blueprint: BlueprintLike | None = None, recording: RecordingStream | None = None, ) -> None: + """ + Connect to a remote Rerun Viewer on the given HTTP(S) URL. + + This function returns immediately. + + Parameters + ---------- + url: + The HTTP(S) URL to connect to + default_blueprint + Optionally set a default blueprint to use for this application. If the application + already has an active blueprint, the new blueprint won't become active until the user + clicks the "reset blueprint" button. If you want to activate the new blueprint + immediately, instead use the [`rerun.send_blueprint`][] API. + recording: + Specifies the [`rerun.RecordingStream`][] to use. + If left unspecified, defaults to the current active data recording, if there is one. + See also: [`rerun.init`][], [`rerun.set_global_data_recording`][]. + + """ + if not _is_connect_grpc_available: raise NotImplementedError("`rerun_sdk` was compiled without `remote` feature, connect_grpc is not available") @@ -156,8 +178,16 @@ def connect_grpc( "No application id found. You must call rerun.init before connecting to a viewer, or provide a recording." ) + # If a blueprint is provided, we need to create a blueprint storage object + blueprint_storage = None + if default_blueprint is not None: + blueprint_storage = create_in_memory_blueprint( + application_id=application_id, blueprint=default_blueprint + ).storage + bindings.connect_grpc( - addr=addr, + url=url, + default_blueprint=blueprint_storage, recording=recording.to_native() if recording is not None else None, ) @@ -520,3 +550,58 @@ def spawn( recording=recording, # NOLINT default_blueprint=default_blueprint, ) + + +def spawn_grpc( + *, + port: int = 1852, + connect: bool = True, + memory_limit: str = "75%", + hide_welcome_screen: bool = False, + default_blueprint: BlueprintLike | None = None, + recording: RecordingStream | None = None, +) -> None: + """ + Spawn a Rerun Viewer, listening on the given port. + + This is often the easiest and best way to use Rerun. + Just call this once at the start of your program. + + You can also call [rerun.init][] with a `spawn=True` argument. + + Parameters + ---------- + port: + The port to listen on. + connect: + also connect to the viewer and stream logging data to it. + memory_limit: + An upper limit on how much memory the Rerun Viewer should use. + When this limit is reached, Rerun will drop the oldest data. + Example: `16GB` or `50%` (of system total). + hide_welcome_screen: + Hide the normal Rerun welcome screen. + recording: + Specifies the [`rerun.RecordingStream`][] to use if `connect = True`. + If left unspecified, defaults to the current active data recording, if there is one. + See also: [`rerun.init`][], [`rerun.set_global_data_recording`][]. + default_blueprint + Optionally set a default blueprint to use for this application. If the application + already has an active blueprint, the new blueprint won't become active until the user + clicks the "reset blueprint" button. If you want to activate the new blueprint + immediately, instead use the [`rerun.send_blueprint`][] API. + + """ + + if not is_recording_enabled(recording): + logging.warning("Rerun is disabled - spawn() call ignored.") + return + + _spawn_viewer(port=port, memory_limit=memory_limit, hide_welcome_screen=hide_welcome_screen) + + if connect: + connect_grpc( + f"http://127.0.0.1:{port}", + recording=recording, # NOLINT + default_blueprint=default_blueprint, + ) From 5f62b431f8e4f093b81eac49d9502f305549c3c4 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Tue, 28 Jan 2025 15:00:46 +0100 Subject: [PATCH 16/87] Connect to `gRPC` on web by default --- .../store/re_data_source/src/data_source.rs | 8 ++- crates/store/re_grpc_server/src/lib.rs | 23 ++++--- crates/store/re_grpc_server/src/main.rs | 3 +- crates/store/re_grpc_server/src/shutdown.rs | 31 +++++++++ crates/top/rerun/src/commands/entrypoint.rs | 69 ++++++++----------- crates/viewer/re_viewer/src/web_tools.rs | 9 +-- 6 files changed, 87 insertions(+), 56 deletions(-) create mode 100644 crates/store/re_grpc_server/src/shutdown.rs diff --git a/crates/store/re_data_source/src/data_source.rs b/crates/store/re_data_source/src/data_source.rs index 26835eb73f20..97c0a0197c52 100644 --- a/crates/store/re_data_source/src/data_source.rs +++ b/crates/store/re_data_source/src/data_source.rs @@ -127,11 +127,13 @@ impl DataSource { } else { // If this is sometyhing like `foo.com` we can't know what it is until we connect to it. // We could/should connect and see what it is, but for now we just take a wild guess instead: - re_log::debug!("Assuming WebSocket endpoint"); + re_log::debug!("Assuming gRPC endpoint"); if !uri.contains("://") { - uri = format!("{}://{uri}", re_ws_comms::PROTOCOL); + // TODO(jan): this should be `https` if it's not localhost, anything hosted over public network + // should be going through https, anyway... + uri = format!("http://{uri}"); } - Self::WebSocketAddr(uri) + Self::MessageProxy { url: uri } } } diff --git a/crates/store/re_grpc_server/src/lib.rs b/crates/store/re_grpc_server/src/lib.rs index 8c74704d7659..e414fb897ea7 100644 --- a/crates/store/re_grpc_server/src/lib.rs +++ b/crates/store/re_grpc_server/src/lib.rs @@ -1,5 +1,7 @@ //! Server implementation of an in-memory Storage Node. +pub mod shutdown; + use std::collections::VecDeque; use std::net::SocketAddr; use std::pin::Pin; @@ -19,6 +21,7 @@ use tokio_stream::Stream; use tokio_stream::StreamExt as _; use tonic::transport::server::TcpIncoming; use tonic::transport::Server; +use tower_http::cors::CorsLayer; pub const DEFAULT_SERVER_PORT: u16 = 1852; pub const DEFAULT_MEMORY_LIMIT: MemoryLimit = MemoryLimit::UNLIMITED; @@ -29,13 +32,15 @@ pub const DEFAULT_MEMORY_LIMIT: MemoryLimit = MemoryLimit::UNLIMITED; pub async fn serve( addr: SocketAddr, memory_limit: MemoryLimit, + shutdown: shutdown::Shutdown, ) -> Result<(), tonic::transport::Error> { - serve_impl(addr, MessageProxy::new(memory_limit)).await + serve_impl(addr, MessageProxy::new(memory_limit), shutdown).await } async fn serve_impl( addr: SocketAddr, message_proxy: MessageProxy, + shutdown: shutdown::Shutdown, ) -> Result<(), tonic::transport::Error> { let tcp_listener = TcpListener::bind(addr) .await @@ -46,12 +51,7 @@ async fn serve_impl( re_log::info!("Listening for gRPC connections on http://{addr}"); - use tower_http::cors::{Any, CorsLayer}; - let cors = CorsLayer::new() - .allow_origin(Any) - .allow_methods(Any) - .allow_headers(Any); - + let cors = CorsLayer::very_permissive(); let grpc_web = tonic_web::GrpcWebLayer::new(); let routes = { @@ -67,13 +67,14 @@ async fn serve_impl( .layer(cors) // Allow CORS requests from web clients .layer(grpc_web) // Support `grpc-web` clients .add_routes(routes) - .serve_with_incoming(incoming) + .serve_with_incoming_shutdown(incoming, shutdown.wait()) .await } pub fn spawn_with_recv( addr: SocketAddr, memory_limit: MemoryLimit, + shutdown: shutdown::Shutdown, ) -> re_smart_channel::Receiver { let (channel_tx, channel_rx) = re_smart_channel::smart_channel( re_smart_channel::SmartMessageSource::MessageProxy { @@ -84,7 +85,11 @@ pub fn spawn_with_recv( }, ); let (message_proxy, mut broadcast_rx) = MessageProxy::new_with_recv(memory_limit); - tokio::spawn(serve_impl(addr, message_proxy)); + tokio::spawn(async move { + if let Err(err) = serve_impl(addr, message_proxy, shutdown).await { + re_log::error!("message proxy server crashed: {err}"); + } + }); tokio::spawn(async move { loop { let msg = match broadcast_rx.recv().await { diff --git a/crates/store/re_grpc_server/src/main.rs b/crates/store/re_grpc_server/src/main.rs index 4f5dd0dbdf38..50071d43fee3 100644 --- a/crates/store/re_grpc_server/src/main.rs +++ b/crates/store/re_grpc_server/src/main.rs @@ -2,7 +2,7 @@ use std::net::Ipv4Addr; use std::net::SocketAddr; use std::net::SocketAddrV4; -use re_grpc_server::{serve, DEFAULT_MEMORY_LIMIT, DEFAULT_SERVER_PORT}; +use re_grpc_server::{serve, shutdown, DEFAULT_MEMORY_LIMIT, DEFAULT_SERVER_PORT}; #[tokio::main(flavor = "current_thread")] async fn main() -> Result<(), tonic::transport::Error> { @@ -14,6 +14,7 @@ async fn main() -> Result<(), tonic::transport::Error> { DEFAULT_SERVER_PORT, )), DEFAULT_MEMORY_LIMIT, + shutdown::never(), ) .await } diff --git a/crates/store/re_grpc_server/src/shutdown.rs b/crates/store/re_grpc_server/src/shutdown.rs new file mode 100644 index 000000000000..01b6c8f74f24 --- /dev/null +++ b/crates/store/re_grpc_server/src/shutdown.rs @@ -0,0 +1,31 @@ +use tokio::sync::oneshot; + +pub fn shutdown() -> (Signal, Shutdown) { + let (tx, rx) = oneshot::channel(); + (Signal(tx), Shutdown(Some(rx))) +} + +pub fn never() -> Shutdown { + Shutdown(None) +} + +pub struct Signal(oneshot::Sender<()>); + +impl Signal { + pub fn stop(self) { + self.0.send(()).ok(); + } +} + +pub struct Shutdown(Option>); + +impl Shutdown { + pub async fn wait(self) { + if let Some(rx) = self.0 { + rx.await.ok(); + } else { + // Never resolve + std::future::pending::<()>().await; + } + } +} diff --git a/crates/top/rerun/src/commands/entrypoint.rs b/crates/top/rerun/src/commands/entrypoint.rs index e42b7b176a38..18e0016ca41a 100644 --- a/crates/top/rerun/src/commands/entrypoint.rs +++ b/crates/top/rerun/src/commands/entrypoint.rs @@ -131,7 +131,7 @@ When persisted, the state will be stored at the following locations: /// What TCP port do we listen to for SDKs to connect to. #[cfg(feature = "server")] - #[clap(long, default_value_t = re_sdk_comms::DEFAULT_SERVER_PORT)] + #[clap(long, default_value_t = re_grpc_server::DEFAULT_SERVER_PORT)] port: u16, /// Start with the puffin profiler running. @@ -693,6 +693,7 @@ fn run_impl( } }; + #[cfg(feature = "server")] let server_addr = std::net::SocketAddr::new(args.bind, args.port); #[cfg(feature = "server")] let server_memory_limit = re_memory::MemoryLimit::parse(&args.server_memory_limit) @@ -744,8 +745,11 @@ fn run_impl( ); is_another_viewer_running = true; } else { - let server: Receiver = - re_grpc_server::spawn_with_recv(server_addr, server_memory_limit); + let server: Receiver = re_grpc_server::spawn_with_recv( + server_addr, + server_memory_limit, + re_grpc_server::shutdown::never(), + ); rxs.push(server); } } @@ -775,8 +779,7 @@ fn run_impl( ); } - #[cfg(feature = "server")] - #[cfg(feature = "web_viewer")] + #[cfg(all(feature = "server", feature = "web_viewer"))] if args.url_or_paths.is_empty() && (args.port == args.web_viewer_port.0 || args.port == args.ws_server_port.0) { @@ -787,44 +790,32 @@ fn run_impl( ); } - #[cfg(feature = "server")] + #[cfg(all(feature = "server", feature = "web_viewer"))] { - // This is the server which the web viewer will talk to: - let _ws_server = re_ws_comms::RerunServer::new( - ReceiveSet::new(rxs), - &args.bind.to_string(), - args.ws_server_port, - server_memory_limit, - )?; - - #[cfg(feature = "web_viewer")] - { - // We always host the web-viewer in case the users wants it, - // but we only open a browser automatically with the `--web-viewer` flag. - - let open_browser = args.web_viewer; - - // This is the server that serves the Wasm+HTML: - WebViewerConfig { - bind_ip: args.bind.to_string(), - web_port: args.web_viewer_port, - source_url: Some(_ws_server.server_url()), - force_wgpu_backend: args.renderer, - video_decoder: args.video_decoder, - open_browser, - } - .host_web_viewer()? - .block(); // dropping should stop the server - } + // We always host the web-viewer in case the users wants it, + // but we only open a browser automatically with the `--web-viewer` flag. + let open_browser = args.web_viewer; - #[cfg(not(feature = "web_viewer"))] - { - // Returning from this function so soon would drop and therefore stop the server. - _ws_server.block(); - } + let url = if server_addr.ip().is_unspecified() || server_addr.ip().is_loopback() { + format!("temp://localhost:{}", server_addr.port()) + } else { + format!("temp://{server_addr}") + }; - return Ok(()); + // This is the server that serves the Wasm+HTML: + WebViewerConfig { + bind_ip: args.bind.to_string(), + web_port: args.web_viewer_port, + source_url: Some(url), + force_wgpu_backend: args.renderer, + video_decoder: args.video_decoder, + open_browser, + } + .host_web_viewer()? + .block(); } + + Ok(()) } else if is_another_viewer_running { // Another viewer is already running on the specified address let url = format!("http://{server_addr}"); diff --git a/crates/viewer/re_viewer/src/web_tools.rs b/crates/viewer/re_viewer/src/web_tools.rs index 259f6173a0eb..ade38ace9158 100644 --- a/crates/viewer/re_viewer/src/web_tools.rs +++ b/crates/viewer/re_viewer/src/web_tools.rs @@ -114,11 +114,12 @@ impl EndpointCategory { } else { // If this is something like `foo.com` we can't know what it is until we connect to it. // We could/should connect and see what it is, but for now we just take a wild guess instead: - re_log::info!("Assuming WebSocket endpoint"); + re_log::info!("Assuming gRPC endpoint"); if uri.contains("://") { - Self::WebSocket(uri) + Self::MessageProxy(uri) } else { - Self::WebSocket(format!("{}://{uri}", re_ws_comms::PROTOCOL)) + // TODO(jan): this should be `https` if it's not localhost or same-origin + Self::MessageProxy(format!("http://{uri}")) } } } @@ -189,7 +190,7 @@ pub fn url_to_receiver( .with_context(|| format!("Failed to connect to WebSocket server at {url}.")), EndpointCategory::MessageProxy(url) => { - re_grpc_client::message_proxy::read::stream(url, Some(ui_waker)) + re_grpc_client::message_proxy::read::stream(&url, Some(ui_waker)) .map_err(|err| err.into()) } } From cd4ee69b39960164655ef06c29897778445828c8 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Tue, 28 Jan 2025 15:00:54 +0100 Subject: [PATCH 17/87] Update quick start connect example --- .../re_viewer/data/quick_start_guides/quick_start_connect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/viewer/re_viewer/data/quick_start_guides/quick_start_connect.py b/crates/viewer/re_viewer/data/quick_start_guides/quick_start_connect.py index 204393418803..bf3d1e6f4b9b 100644 --- a/crates/viewer/re_viewer/data/quick_start_guides/quick_start_connect.py +++ b/crates/viewer/re_viewer/data/quick_start_guides/quick_start_connect.py @@ -7,7 +7,7 @@ rr.init("rerun_example_quick_start_connect") # Connect to a local viewer using the default port -rr.connect_tcp() +rr.connect_grpc() # Create some data From 4e36f33e018735f2a27e38b7f2c23aaa30996093 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Tue, 28 Jan 2025 15:13:30 +0100 Subject: [PATCH 18/87] Fix missing header decls (spawn, connect) --- rerun_cpp/src/rerun/c/rerun.h | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/rerun_cpp/src/rerun/c/rerun.h b/rerun_cpp/src/rerun/c/rerun.h index 94f1972176ac..ddf8ed000591 100644 --- a/rerun_cpp/src/rerun/c/rerun.h +++ b/rerun_cpp/src/rerun/c/rerun.h @@ -427,6 +427,21 @@ extern void rr_recording_stream_connect( rr_recording_stream stream, rr_string tcp_addr, float flush_timeout_sec, rr_error* error ); +/// Connect to a remote Rerun Viewer on the given URL. +/// +/// Requires that you first start a Rerun Viewer by typing 'rerun' in a terminal. +/// +/// flush_timeout_sec: +/// The minimum time the SDK will wait during a flush before potentially +/// dropping data if progress is not being made. Passing a negative value indicates no timeout, +/// and can cause a call to `flush` to block indefinitely. +/// +/// This function returns immediately and will only raise an error for argument parsing errors, +/// not for connection errors as these happen asynchronously. +extern void rr_recording_stream_connect_grpc( + rr_recording_stream stream, rr_string url, rr_error* error +); + /// Spawns a new Rerun Viewer process from an executable available in PATH, then connects to it /// over TCP. /// @@ -449,6 +464,22 @@ extern void rr_recording_stream_spawn( rr_error* error ); +/// Spawns a new Rerun Viewer process from an executable available in PATH, then connects to it +/// over gRPC. +/// +/// This function returns immediately and will only raise an error for argument parsing errors, +/// not for connection errors as these happen asynchronously. +/// +/// ## Parameters +/// +/// spawn_opts: +/// Configuration of the spawned process. +/// Refer to `rr_spawn_options` documentation for details. +/// Passing null is valid and will result in the recommended defaults. +extern void rr_recording_stream_spawn_grpc( + rr_recording_stream stream, const rr_spawn_options* spawn_opts, rr_error* error +); + /// Stream all log-data to a given `.rrd` file. /// /// This function returns immediately. From 3a586b75dcb977fb13d2231396582aeb69f6c89e Mon Sep 17 00:00:00 2001 From: jprochazk Date: Tue, 28 Jan 2025 15:15:35 +0100 Subject: [PATCH 19/87] Fix comment --- crates/store/re_data_source/src/data_source.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/store/re_data_source/src/data_source.rs b/crates/store/re_data_source/src/data_source.rs index 97c0a0197c52..da170151c2de 100644 --- a/crates/store/re_data_source/src/data_source.rs +++ b/crates/store/re_data_source/src/data_source.rs @@ -130,7 +130,7 @@ impl DataSource { re_log::debug!("Assuming gRPC endpoint"); if !uri.contains("://") { // TODO(jan): this should be `https` if it's not localhost, anything hosted over public network - // should be going through https, anyway... + // should be going through https, anyway. uri = format!("http://{uri}"); } Self::MessageProxy { url: uri } From 875b57bacc72caf07a69c4b81df2ae161acaf0a6 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Tue, 28 Jan 2025 15:16:27 +0100 Subject: [PATCH 20/87] Run `py-fmt` --- rerun_py/rerun_sdk/rerun/blueprint/api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/rerun_py/rerun_sdk/rerun/blueprint/api.py b/rerun_py/rerun_sdk/rerun/blueprint/api.py index 4a9af2f10e21..1326c250c8dc 100644 --- a/rerun_py/rerun_sdk/rerun/blueprint/api.py +++ b/rerun_py/rerun_sdk/rerun/blueprint/api.py @@ -724,7 +724,6 @@ def spawn_grpc( self.connect_grpc(application_id=application_id, url=f"http://127.0.0.1:{port}") - BlueprintLike = Union[Blueprint, View, Container] """ A type that can be converted to a blueprint. From 742d5aa6744f1972144764cb5607a6765742a656 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Tue, 28 Jan 2025 15:16:33 +0100 Subject: [PATCH 21/87] Remove unused dependencies --- Cargo.lock | 2 -- crates/top/rerun-cli/Cargo.toml | 5 +---- crates/top/rerun/Cargo.toml | 4 +--- crates/viewer/re_viewer/Cargo.toml | 1 - 4 files changed, 2 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dbf94a3936e6..c43e5f4aa6db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7001,7 +7001,6 @@ dependencies = [ "re_viewer_context", "re_viewport", "re_viewport_blueprint", - "re_ws_comms", "rfd", "ron", "serde", @@ -7368,7 +7367,6 @@ dependencies = [ "re_log_types", "re_memory", "re_sdk", - "re_sdk_comms", "re_smart_channel", "re_tracing", "re_types", diff --git a/crates/top/rerun-cli/Cargo.toml b/crates/top/rerun-cli/Cargo.toml index 9f20966be467..284d9df52a45 100644 --- a/crates/top/rerun-cli/Cargo.toml +++ b/crates/top/rerun-cli/Cargo.toml @@ -99,10 +99,7 @@ rerun = { workspace = true, features = [ document-features.workspace = true mimalloc = "0.1.43" -tokio = { workspace = true, features = [ - "macros", - "rt-multi-thread", -] } +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } [build-dependencies] diff --git a/crates/top/rerun/Cargo.toml b/crates/top/rerun/Cargo.toml index 838054c664fd..2d3089ea5176 100644 --- a/crates/top/rerun/Cargo.toml +++ b/crates/top/rerun/Cargo.toml @@ -101,12 +101,11 @@ run = [ "dep:re_data_source", "re_log_encoding/encoder", "re_log_encoding/decoder", - "dep:re_sdk_comms", "dep:re_ws_comms", ] ## Support for running a TCP server that listens to incoming log messages from a Rerun SDK. -server = ["re_sdk_comms?/server", "dep:re_grpc_server"] +server = ["dep:re_grpc_server"] ## Embed the Rerun SDK & built-in types and re-export all of their public symbols. sdk = ["dep:re_sdk", "dep:re_types"] @@ -154,7 +153,6 @@ re_data_source = { workspace = true, optional = true } re_dataframe = { workspace = true, optional = true } re_grpc_server = { workspace = true, optional = true } re_sdk = { workspace = true, optional = true } -re_sdk_comms = { workspace = true, optional = true } re_types = { workspace = true, optional = true } re_viewer = { workspace = true, optional = true } re_web_viewer_server = { workspace = true, optional = true } diff --git a/crates/viewer/re_viewer/Cargo.toml b/crates/viewer/re_viewer/Cargo.toml index 150edd40a2c2..1176087a5185 100644 --- a/crates/viewer/re_viewer/Cargo.toml +++ b/crates/viewer/re_viewer/Cargo.toml @@ -90,7 +90,6 @@ re_video.workspace = true re_viewer_context.workspace = true re_viewport_blueprint.workspace = true re_viewport.workspace = true -re_ws_comms = { workspace = true, features = ["client"] } # Internal (optional): re_analytics = { workspace = true, optional = true } From eff83f2e2133d6307e29d830775f69b76e210dde Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 29 Jan 2025 11:33:13 +0100 Subject: [PATCH 22/87] Remove all remaining usage of `re_ws_comms` --- crates/store/re_data_source/Cargo.toml | 1 - .../store/re_data_source/src/data_source.rs | 12 --- crates/store/re_data_source/src/lib.rs | 2 - crates/store/re_grpc_server/Cargo.toml | 1 + crates/store/re_grpc_server/src/lib.rs | 67 ++++++++++++++++- crates/store/re_grpc_server/src/shutdown.rs | 17 ++++- crates/top/re_sdk/Cargo.toml | 5 +- crates/top/re_sdk/src/recording_stream.rs | 10 +-- crates/top/re_sdk/src/web_viewer.rs | 75 ++++++++++++------- crates/top/rerun/Cargo.toml | 2 - crates/top/rerun/src/commands/entrypoint.rs | 14 +--- .../quick_start_guides/quick_start_spawn.py | 3 +- rerun_py/Cargo.toml | 5 +- rerun_py/rerun_sdk/rerun/sinks.py | 16 ++-- rerun_py/src/python_bridge.rs | 10 +-- scripts/run_all.py | 2 +- 16 files changed, 150 insertions(+), 92 deletions(-) diff --git a/crates/store/re_data_source/Cargo.toml b/crates/store/re_data_source/Cargo.toml index b47ea36006a1..a579a8775709 100644 --- a/crates/store/re_data_source/Cargo.toml +++ b/crates/store/re_data_source/Cargo.toml @@ -37,7 +37,6 @@ re_log_types.workspace = true re_log.workspace = true re_smart_channel.workspace = true re_tracing.workspace = true -re_ws_comms = { workspace = true, features = ["client"] } anyhow.workspace = true itertools.workspace = true diff --git a/crates/store/re_data_source/src/data_source.rs b/crates/store/re_data_source/src/data_source.rs index da170151c2de..6bc4c1468cf8 100644 --- a/crates/store/re_data_source/src/data_source.rs +++ b/crates/store/re_data_source/src/data_source.rs @@ -24,9 +24,6 @@ pub enum DataSource { /// This is what you get when loading a file on Web, or when using drag-n-drop. FileContents(re_log_types::FileSource, FileContents), - /// A remote Rerun server. - WebSocketAddr(String), - // RRD data streaming in from standard input. #[cfg(not(target_arch = "wasm32"))] Stdin, @@ -113,10 +110,6 @@ impl DataSource { url: uri, follow: false, } - } else if uri.starts_with("ws://") || uri.starts_with("wss://") { - Self::WebSocketAddr(uri) - - // Now we are into heuristics territory: } else if looks_like_a_file_path(&uri) { Self::FilePath(file_source, path) } else if uri.ends_with(".rrd") || uri.ends_with(".rbl") { @@ -143,7 +136,6 @@ impl DataSource { #[cfg(not(target_arch = "wasm32"))] Self::FilePath(_, path) => path.file_name().map(|s| s.to_string_lossy().to_string()), Self::FileContents(_, file_contents) => Some(file_contents.name.clone()), - Self::WebSocketAddr(_) => None, #[cfg(not(target_arch = "wasm32"))] Self::Stdin => None, #[cfg(feature = "grpc")] @@ -236,10 +228,6 @@ impl DataSource { Ok(rx) } - Self::WebSocketAddr(rerun_server_ws_url) => { - crate::web_sockets::connect_to_ws_url(&rerun_server_ws_url, on_msg) - } - #[cfg(not(target_arch = "wasm32"))] Self::Stdin => { let (tx, rx) = re_smart_channel::smart_channel( diff --git a/crates/store/re_data_source/src/lib.rs b/crates/store/re_data_source/src/lib.rs index 996777f84201..bec3bb0c190a 100644 --- a/crates/store/re_data_source/src/lib.rs +++ b/crates/store/re_data_source/src/lib.rs @@ -7,13 +7,11 @@ //! Also handles different file types: rrd, images, text files, 3D models, point clouds… mod data_source; -mod web_sockets; #[cfg(not(target_arch = "wasm32"))] mod load_stdin; pub use self::data_source::DataSource; -pub use self::web_sockets::connect_to_ws_url; // ---------------------------------------------------------------------------- diff --git a/crates/store/re_grpc_server/Cargo.toml b/crates/store/re_grpc_server/Cargo.toml index 1d868cd7bcb7..6d1d44e7ebec 100644 --- a/crates/store/re_grpc_server/Cargo.toml +++ b/crates/store/re_grpc_server/Cargo.toml @@ -34,6 +34,7 @@ re_tracing.workspace = true re_types.workspace = true # External +parking_lot.workspace = true tokio.workspace = true tokio-stream = { workspace = true, features = ["sync"] } tokio-util.workspace = true diff --git a/crates/store/re_grpc_server/src/lib.rs b/crates/store/re_grpc_server/src/lib.rs index e414fb897ea7..0885261b3d34 100644 --- a/crates/store/re_grpc_server/src/lib.rs +++ b/crates/store/re_grpc_server/src/lib.rs @@ -71,6 +71,69 @@ async fn serve_impl( .await } +pub async fn serve_with_send( + addr: SocketAddr, + memory_limit: MemoryLimit, + shutdown: shutdown::Shutdown, + channel_rx: re_smart_channel::Receiver, +) { + let message_proxy = MessageProxy::new(memory_limit); + let event_tx = message_proxy.event_tx.clone(); + + tokio::spawn(async move { + use re_smart_channel::SmartMessagePayload; + + loop { + let msg = match channel_rx.try_recv() { + Ok(msg) => match msg.payload { + SmartMessagePayload::Msg(msg) => msg, + SmartMessagePayload::Flush { on_flush_done } => { + on_flush_done(); // we don't buffer + continue; + } + SmartMessagePayload::Quit(err) => { + if let Some(err) = err { + re_log::debug!("smart channel sender quit: {err}"); + } else { + re_log::debug!("smart channel sender quit"); + } + break; + } + }, + Err(re_smart_channel::TryRecvError::Disconnected) => { + re_log::debug!("smart channel sender closed, closing receiver"); + break; + } + Err(re_smart_channel::TryRecvError::Empty) => { + // Let other tokio tasks run: + tokio::task::yield_now().await; + continue; + } + }; + + let msg = match re_log_encoding::protobuf_conversions::log_msg_to_proto( + msg, + re_log_encoding::Compression::LZ4, + ) { + Ok(msg) => msg, + Err(err) => { + re_log::error!("failed to encode message: {err}"); + continue; + } + }; + + if event_tx.send(Event::Message(msg)).await.is_err() { + re_log::debug!("shut down, closing sender"); + break; + } + } + }); + + if let Err(err) = serve_impl(addr, message_proxy, shutdown).await { + re_log::error!("message proxy server crashed: {err}"); + } +} + pub fn spawn_with_recv( addr: SocketAddr, memory_limit: MemoryLimit, @@ -289,9 +352,7 @@ impl MessageProxy { Self::new_with_recv(server_memory_limit).0 } - pub fn new_with_recv( - server_memory_limit: MemoryLimit, - ) -> (Self, broadcast::Receiver) { + fn new_with_recv(server_memory_limit: MemoryLimit) -> (Self, broadcast::Receiver) { // Channel capacity is completely arbitrary. // We just want something large enough to handle bursts of messages. let (event_tx, event_rx) = mpsc::channel(1024); diff --git a/crates/store/re_grpc_server/src/shutdown.rs b/crates/store/re_grpc_server/src/shutdown.rs index 01b6c8f74f24..77dda02230c6 100644 --- a/crates/store/re_grpc_server/src/shutdown.rs +++ b/crates/store/re_grpc_server/src/shutdown.rs @@ -1,25 +1,34 @@ +use parking_lot::Mutex; use tokio::sync::oneshot; pub fn shutdown() -> (Signal, Shutdown) { let (tx, rx) = oneshot::channel(); - (Signal(tx), Shutdown(Some(rx))) + (Signal(Mutex::new(Some(tx))), Shutdown(Some(rx))) } pub fn never() -> Shutdown { Shutdown(None) } -pub struct Signal(oneshot::Sender<()>); +pub struct Signal(Mutex>>); impl Signal { - pub fn stop(self) { - self.0.send(()).ok(); + /// Ask the server to shut down. + /// + /// Subsequent calls to this function have no effect. + pub fn stop(&self) { + if let Some(sender) = self.0.lock().take() { + sender.send(()).ok(); + } } } pub struct Shutdown(Option>); impl Shutdown { + /// Returns a future that resolves when the signal is sent. + /// + /// If this was constructed with [`never`], then it never resolves. pub async fn wait(self) { if let Some(rx) = self.0 { rx.await.ok(); diff --git a/crates/top/re_sdk/Cargo.toml b/crates/top/re_sdk/Cargo.toml index 12583925c34d..4705f8205bba 100644 --- a/crates/top/re_sdk/Cargo.toml +++ b/crates/top/re_sdk/Cargo.toml @@ -42,9 +42,8 @@ data_loaders = ["dep:re_data_loader", "dep:re_smart_channel"] web_viewer = [ "dep:re_smart_channel", "dep:re_web_viewer_server", - "dep:re_ws_comms", + "dep:tokio", "dep:webbrowser", - "re_ws_comms?/server", ] grpc = ["re_grpc_client/redap"] @@ -76,9 +75,9 @@ thiserror.workspace = true re_data_loader = { workspace = true, optional = true } re_smart_channel = { workspace = true, optional = true } -re_ws_comms = { workspace = true, optional = true } re_web_viewer_server = { workspace = true, optional = true } +tokio = { workspace = true, optional = true } webbrowser = { workspace = true, optional = true } # Native unix dependencies: diff --git a/crates/top/re_sdk/src/recording_stream.rs b/crates/top/re_sdk/src/recording_stream.rs index 3717c6d45592..a3e61c4d83bc 100644 --- a/crates/top/re_sdk/src/recording_stream.rs +++ b/crates/top/re_sdk/src/recording_stream.rs @@ -22,8 +22,6 @@ use re_types_core::{AsComponents, SerializationError, SerializedComponentColumn} #[cfg(feature = "web_viewer")] use re_web_viewer_server::WebViewerServerPort; -#[cfg(feature = "web_viewer")] -use re_ws_comms::RerunServerPort; use crate::binary_stream_sink::BinaryStreamStorage; use crate::sink::{LogSink, MemorySinkStorage}; @@ -649,14 +647,14 @@ impl RecordingStreamBuilder { self, bind_ip: &str, web_port: WebViewerServerPort, - ws_port: RerunServerPort, + grpc_port: u16, server_memory_limit: re_memory::MemoryLimit, open_browser: bool, ) -> RecordingStreamResult { self.serve_web( bind_ip, web_port, - ws_port, + grpc_port, server_memory_limit, open_browser, ) @@ -697,7 +695,7 @@ impl RecordingStreamBuilder { self, bind_ip: &str, web_port: WebViewerServerPort, - ws_port: RerunServerPort, + grpc_port: u16, server_memory_limit: re_memory::MemoryLimit, open_browser: bool, ) -> RecordingStreamResult { @@ -707,7 +705,7 @@ impl RecordingStreamBuilder { open_browser, bind_ip, web_port, - ws_port, + grpc_port, server_memory_limit, )?; RecordingStream::new(store_info, batcher_config, sink) diff --git a/crates/top/re_sdk/src/web_viewer.rs b/crates/top/re_sdk/src/web_viewer.rs index 44b35bf42577..75c02a1cb264 100644 --- a/crates/top/re_sdk/src/web_viewer.rs +++ b/crates/top/re_sdk/src/web_viewer.rs @@ -1,6 +1,5 @@ use re_log_types::LogMsg; use re_web_viewer_server::{WebViewerServer, WebViewerServerError, WebViewerServerPort}; -use re_ws_comms::{RerunServer, RerunServerPort}; // ---------------------------------------------------------------------------- @@ -11,22 +10,25 @@ pub enum WebViewerSinkError { #[error(transparent)] WebViewerServer(#[from] WebViewerServerError), - /// Failure to host the Rerun WebSocket server. + /// Invalid host IP. #[error(transparent)] - RerunServer(#[from] re_ws_comms::RerunServerError), + InvalidAddress(#[from] std::net::AddrParseError), } /// A [`crate::sink::LogSink`] tied to a hosted Rerun web viewer. This internally stores two servers: -/// * A [`re_ws_comms::RerunServer`] to relay messages from the sink to a websocket connection +/// * A gRPC server to relay messages from the sink to any connected web viewers /// * A [`WebViewerServer`] to serve the Wasm+HTML struct WebViewerSink { open_browser: bool, - /// Sender to send messages to the [`re_ws_comms::RerunServer`] + /// Sender to send messages to the gRPC server. sender: re_smart_channel::Sender, + /// The gRPC server thread. + _server_handle: std::thread::JoinHandle<()>, + /// Rerun websocket server. - rerun_server: RerunServer, + server_signal: re_grpc_server::shutdown::Signal, /// The http server serving wasm & html. _webviewer_server: WebViewerServer, @@ -38,26 +40,43 @@ impl WebViewerSink { open_browser: bool, bind_ip: &str, web_port: WebViewerServerPort, - ws_port: RerunServerPort, + grpc_port: u16, server_memory_limit: re_memory::MemoryLimit, ) -> Result { - // TODO(cmc): the sources here probably don't make much sense… - let (rerun_tx, rerun_rx) = re_smart_channel::smart_channel( - re_smart_channel::SmartMessageSource::Sdk, + let (signal, shutdown) = re_grpc_server::shutdown::shutdown(); + + let grpc_server_addr = format!("{bind_ip}:{grpc_port}").parse()?; + let (channel_tx, channel_rx) = re_smart_channel::smart_channel::( + re_smart_channel::SmartMessageSource::MessageProxy { + url: format!("http://{grpc_server_addr}"), + }, re_smart_channel::SmartChannelSource::Sdk, ); - - let rerun_server = RerunServer::new( - re_smart_channel::ReceiveSet::new(vec![rerun_rx]), - bind_ip, - ws_port, - server_memory_limit, - )?; + let server_handle = std::thread::Builder::new() + .name("message_proxy_server".to_owned()) + .spawn(move || { + let mut builder = tokio::runtime::Builder::new_current_thread(); + builder.enable_all(); + let rt = builder.build().expect("failed to build tokio runtime"); + + rt.block_on(re_grpc_server::serve_with_send( + grpc_server_addr, + server_memory_limit, + shutdown, + channel_rx, + )); + }) + .expect("failed to spawn thread for message proxy server"); let webviewer_server = WebViewerServer::new(bind_ip, web_port)?; let http_web_viewer_url = webviewer_server.server_url(); - let ws_server_url = rerun_server.server_url(); - let viewer_url = format!("{http_web_viewer_url}?url={ws_server_url}"); + + let viewer_url = + if grpc_server_addr.ip().is_unspecified() || grpc_server_addr.ip().is_loopback() { + format!("{http_web_viewer_url}?url=temp://localhost:{grpc_port}") + } else { + format!("{http_web_viewer_url}?url=temp://{grpc_server_addr}") + }; re_log::info!("Hosting a web-viewer at {viewer_url}"); if open_browser { @@ -66,8 +85,9 @@ impl WebViewerSink { Ok(Self { open_browser, - sender: rerun_tx, - rerun_server, + sender: channel_tx, + _server_handle: server_handle, + server_signal: signal, _webviewer_server: webviewer_server, }) } @@ -90,7 +110,7 @@ impl crate::sink::LogSink for WebViewerSink { impl Drop for WebViewerSink { fn drop(&mut self) { - if self.open_browser && self.rerun_server.num_accepted_clients() == 0 { + if self.open_browser { // For small scripts that execute fast we run the risk of finishing // before the browser has a chance to connect. // Let's give it a little more time: @@ -98,9 +118,7 @@ impl Drop for WebViewerSink { std::thread::sleep(std::time::Duration::from_millis(1000)); } - if self.rerun_server.num_accepted_clients() == 0 { - re_log::info!("Shutting down without any clients ever having connected. Consider sleeping to give them more time to connect"); - } + self.server_signal.stop(); } } @@ -119,9 +137,10 @@ pub struct WebViewerConfig { /// Defaults to [`WebViewerServerPort::AUTO`]. pub web_port: WebViewerServerPort, + // TODO(#8761): URL prefix /// The url from which a spawned webviewer should source /// - /// This url could be a hosted RRD file or a `ws://` url to a running [`re_ws_comms::RerunServer`]. + /// This url could be a hosted RRD file or a `temp://` url to a running gRPC server. /// Has no effect if [`Self::open_browser`] is false. pub source_url: Option, @@ -226,14 +245,14 @@ pub fn new_sink( open_browser: bool, bind_ip: &str, web_port: WebViewerServerPort, - ws_port: RerunServerPort, + grpc_port: u16, server_memory_limit: re_memory::MemoryLimit, ) -> Result, WebViewerSinkError> { Ok(Box::new(WebViewerSink::new( open_browser, bind_ip, web_port, - ws_port, + grpc_port, server_memory_limit, )?)) } diff --git a/crates/top/rerun/Cargo.toml b/crates/top/rerun/Cargo.toml index 2d3089ea5176..55b86921111d 100644 --- a/crates/top/rerun/Cargo.toml +++ b/crates/top/rerun/Cargo.toml @@ -101,7 +101,6 @@ run = [ "dep:re_data_source", "re_log_encoding/encoder", "re_log_encoding/decoder", - "dep:re_ws_comms", ] ## Support for running a TCP server that listens to incoming log messages from a Rerun SDK. @@ -156,7 +155,6 @@ re_sdk = { workspace = true, optional = true } re_types = { workspace = true, optional = true } re_viewer = { workspace = true, optional = true } re_web_viewer_server = { workspace = true, optional = true } -re_ws_comms = { workspace = true, optional = true, features = ["server"] } env_logger = { workspace = true, optional = true } log = { workspace = true, optional = true } diff --git a/crates/top/rerun/src/commands/entrypoint.rs b/crates/top/rerun/src/commands/entrypoint.rs index 18e0016ca41a..4356bdd46b91 100644 --- a/crates/top/rerun/src/commands/entrypoint.rs +++ b/crates/top/rerun/src/commands/entrypoint.rs @@ -14,8 +14,6 @@ use crate::{commands::RrdCommands, CallSource}; use re_sdk::web_viewer::WebViewerConfig; #[cfg(feature = "web_viewer")] use re_web_viewer_server::WebViewerServerPort; -#[cfg(feature = "server")] -use re_ws_comms::RerunServerPort; #[cfg(feature = "analytics")] use crate::commands::AnalyticsCommands; @@ -129,7 +127,7 @@ When persisted, the state will be stored at the following locations: )] persist_state: bool, - /// What TCP port do we listen to for SDKs to connect to. + /// What port do we listen to for SDKs to connect to over gRPC. #[cfg(feature = "server")] #[clap(long, default_value_t = re_grpc_server::DEFAULT_SERVER_PORT)] port: u16, @@ -222,12 +220,6 @@ If no arguments are given, a server will be hosted which a Rerun SDK can connect #[clap(long)] window_size: Option, - /// What port do we listen to for incoming websocket connections from the viewer. - /// A port of 0 will pick a random port. - #[cfg(feature = "server")] - #[clap(long, default_value_t = Default::default())] - ws_server_port: RerunServerPort, - /// Override the default graphics backend and for a specific one instead. /// /// When using `--web-viewer` this should be one of: `webgpu`, `webgl`. @@ -780,9 +772,7 @@ fn run_impl( } #[cfg(all(feature = "server", feature = "web_viewer"))] - if args.url_or_paths.is_empty() - && (args.port == args.web_viewer_port.0 || args.port == args.ws_server_port.0) - { + if args.url_or_paths.is_empty() && (args.port == args.web_viewer_port.0) { anyhow::bail!( "Trying to spawn a websocket server on {}, but this port is \ already used by the server we're connecting to. Please specify a different port.", diff --git a/crates/viewer/re_viewer/data/quick_start_guides/quick_start_spawn.py b/crates/viewer/re_viewer/data/quick_start_guides/quick_start_spawn.py index d108fc4d6d08..f9bf11fcf17a 100644 --- a/crates/viewer/re_viewer/data/quick_start_guides/quick_start_spawn.py +++ b/crates/viewer/re_viewer/data/quick_start_guides/quick_start_spawn.py @@ -4,7 +4,8 @@ import rerun as rr # Initialize the SDK, give our recording a unique name, and spawn a viewer -rr.init("rerun_example_quick_start_spawn", spawn=True) +rr.init("rerun_example_quick_start_spawn") +rr.serve_web(open_browser=True) # Create some data SIZE = 10 diff --git a/rerun_py/Cargo.toml b/rerun_py/Cargo.toml index dfa7be8beee1..60717fd03466 100644 --- a/rerun_py/Cargo.toml +++ b/rerun_py/Cargo.toml @@ -38,7 +38,6 @@ nasm = ["re_video/nasm"] remote = [ "dep:object_store", "dep:re_protos", - "dep:re_ws_comms", "dep:re_grpc_client", "dep:tokio", "dep:tokio-stream", @@ -54,7 +53,7 @@ remote = [ web_viewer = [ "re_sdk/web_viewer", "dep:re_web_viewer_server", - "dep:re_ws_comms", + "dep:re_grpc_server", ] @@ -65,6 +64,7 @@ re_chunk.workspace = true re_chunk_store.workspace = true re_dataframe.workspace = true re_grpc_client = { workspace = true, optional = true } +re_grpc_server = { workspace = true, optional = true } re_log = { workspace = true, features = ["setup"] } re_log_encoding = { workspace = true } re_log_types.workspace = true @@ -73,7 +73,6 @@ re_sdk = { workspace = true, features = ["data_loaders"] } re_sorbet.workspace = true re_video.workspace = true re_web_viewer_server = { workspace = true, optional = true } -re_ws_comms = { workspace = true, optional = true } arrow = { workspace = true, features = ["pyarrow"] } crossbeam.workspace = true diff --git a/rerun_py/rerun_sdk/rerun/sinks.py b/rerun_py/rerun_sdk/rerun/sinks.py index 7c90aac6832f..557c49fe34a7 100644 --- a/rerun_py/rerun_sdk/rerun/sinks.py +++ b/rerun_py/rerun_sdk/rerun/sinks.py @@ -321,7 +321,7 @@ def serve( *, open_browser: bool = True, web_port: int | None = None, - ws_port: int | None = None, + grpc_port: int | None = None, default_blueprint: BlueprintLike | None = None, recording: RecordingStream | None = None, server_memory_limit: str = "25%", @@ -348,8 +348,8 @@ def serve( Open the default browser to the viewer. web_port: The port to serve the web viewer on (defaults to 9090). - ws_port: - The port to serve the WebSocket server on (defaults to 9877) + grpc_port: + The port to serve the gRPC server on (defaults to 1852) default_blueprint: Optionally set a default blueprint to use for this application. If the application already has an active blueprint, the new blueprint won't become active until the user @@ -373,7 +373,7 @@ def serve( return serve_web( open_browser=open_browser, web_port=web_port, - ws_port=ws_port, + grpc_port=grpc_port, default_blueprint=default_blueprint, recording=recording, # NOLINT server_memory_limit=server_memory_limit, @@ -384,7 +384,7 @@ def serve_web( *, open_browser: bool = True, web_port: int | None = None, - ws_port: int | None = None, + grpc_port: int | None = None, default_blueprint: BlueprintLike | None = None, recording: RecordingStream | None = None, server_memory_limit: str = "25%", @@ -407,8 +407,8 @@ def serve_web( Open the default browser to the viewer. web_port: The port to serve the web viewer on (defaults to 9090). - ws_port: - The port to serve the WebSocket server on (defaults to 9877) + grpc_port: + The port to serve the gRPC server on (defaults to 1852) default_blueprint: Optionally set a default blueprint to use for this application. If the application already has an active blueprint, the new blueprint won't become active until the user @@ -445,7 +445,7 @@ def serve_web( bindings.serve_web( open_browser, web_port, - ws_port, + grpc_port, server_memory_limit=server_memory_limit, default_blueprint=blueprint_storage, recording=recording.to_native() if recording is not None else None, diff --git a/rerun_py/src/python_bridge.rs b/rerun_py/src/python_bridge.rs index 3cc7f5a14987..d1deaf44f8ff 100644 --- a/rerun_py/src/python_bridge.rs +++ b/rerun_py/src/python_bridge.rs @@ -27,8 +27,6 @@ use re_sdk::{ #[cfg(feature = "web_viewer")] use re_web_viewer_server::WebViewerServerPort; -#[cfg(feature = "web_viewer")] -use re_ws_comms::RerunServerPort; // --- FFI --- @@ -1033,11 +1031,11 @@ impl PyBinarySinkStorage { /// Serve a web-viewer. #[allow(clippy::unnecessary_wraps)] // False positive #[pyfunction] -#[pyo3(signature = (open_browser, web_port, ws_port, server_memory_limit, default_blueprint = None, recording = None))] +#[pyo3(signature = (open_browser, web_port, grpc_port, server_memory_limit, default_blueprint = None, recording = None))] fn serve_web( open_browser: bool, web_port: Option, - ws_port: Option, + grpc_port: Option, server_memory_limit: String, default_blueprint: Option<&PyMemorySinkStorage>, recording: Option<&PyRecordingStream>, @@ -1060,7 +1058,7 @@ fn serve_web( open_browser, "0.0.0.0", web_port.map(WebViewerServerPort).unwrap_or_default(), - ws_port.map(RerunServerPort).unwrap_or_default(), + grpc_port.unwrap_or(re_grpc_server::DEFAULT_SERVER_PORT), server_memory_limit, ) .map_err(|err| PyRuntimeError::new_err(err.to_string()))?; @@ -1079,7 +1077,7 @@ fn serve_web( _ = default_blueprint; _ = recording; _ = web_port; - _ = ws_port; + _ = grpc_port; _ = open_browser; _ = server_memory_limit; Err(PyRuntimeError::new_err( diff --git a/scripts/run_all.py b/scripts/run_all.py index 19fff88146b6..5b15aa80b3ff 100755 --- a/scripts/run_all.py +++ b/scripts/run_all.py @@ -180,7 +180,7 @@ def start(self) -> Viewer: args += [ "--web-viewer", f"--web-viewer-port={self.web_viewer_port}", - f"--ws-server-port={self.ws_server_port}", + f"--port={self.ws_server_port}", ] self.process = subprocess.Popen(args) From 0cb22dd0bafdb7687bc3ea40fab0f0c30d5906c2 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 29 Jan 2025 11:33:25 +0100 Subject: [PATCH 23/87] Remove `re_ws_comms` --- ARCHITECTURE.md | 1 - Cargo.lock | 75 +-- Cargo.toml | 1 - .../store/re_data_source/src/web_sockets.rs | 43 -- crates/store/re_ws_comms/Cargo.toml | 61 --- crates/store/re_ws_comms/README.md | 10 - crates/store/re_ws_comms/src/client.rs | 60 --- crates/store/re_ws_comms/src/lib.rs | 112 ----- crates/store/re_ws_comms/src/server.rs | 444 ------------------ 9 files changed, 3 insertions(+), 804 deletions(-) delete mode 100644 crates/store/re_data_source/src/web_sockets.rs delete mode 100644 crates/store/re_ws_comms/Cargo.toml delete mode 100644 crates/store/re_ws_comms/README.md delete mode 100644 crates/store/re_ws_comms/src/client.rs delete mode 100644 crates/store/re_ws_comms/src/lib.rs delete mode 100644 crates/store/re_ws_comms/src/server.rs diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index f9bbcffd04eb..6fabb520e56c 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -196,7 +196,6 @@ Update instructions: | re_grpc_server | Host an in-memory Storage Node | | re_sdk_comms | TCP communication between Rerun SDK and Rerun Server | | re_web_viewer_server | Serves the Rerun web viewer (Wasm and HTML) over HTTP | -| re_ws_comms | WebSocket communication library (encoding, decoding, client, server) between a Rerun server and Viewer | ### Build support diff --git a/Cargo.lock b/Cargo.lock index 7bcaf73081a3..7246a2419063 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1805,12 +1805,6 @@ dependencies = [ "syn 2.0.87", ] -[[package]] -name = "data-encoding" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" - [[package]] name = "data-url" version = "0.3.1" @@ -2391,21 +2385,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "ewebsock" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "679247b4a005c82218a5f13b713239b0b6d484ec25347a719f5b7066152a748a" -dependencies = [ - "document-features", - "js-sys", - "log", - "tungstenite", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - [[package]] name = "extend_viewer_ui" version = "0.22.0-alpha.1+dev" @@ -5936,7 +5915,6 @@ dependencies = [ "re_log_types", "re_smart_channel", "re_tracing", - "re_ws_comms", ] [[package]] @@ -6112,6 +6090,7 @@ dependencies = [ name = "re_grpc_server" version = "0.22.0-alpha.1+dev" dependencies = [ + "parking_lot", "re_build_info", "re_byte_size", "re_chunk", @@ -6463,9 +6442,9 @@ dependencies = [ "re_smart_channel", "re_types_core", "re_web_viewer_server", - "re_ws_comms", "similar-asserts", "thiserror 1.0.65", + "tokio", "webbrowser", ] @@ -7200,26 +7179,6 @@ dependencies = [ "tiny_http", ] -[[package]] -name = "re_ws_comms" -version = "0.22.0-alpha.1+dev" -dependencies = [ - "anyhow", - "bincode", - "document-features", - "ewebsock", - "parking_lot", - "polling", - "re_format", - "re_log", - "re_log_types", - "re_memory", - "re_smart_channel", - "re_tracing", - "thiserror 1.0.65", - "tungstenite", -] - [[package]] name = "redox_syscall" version = "0.4.1" @@ -7436,7 +7395,6 @@ dependencies = [ "re_video", "re_viewer", "re_web_viewer_server", - "re_ws_comms", "similar-asserts", "tokio", "unindent", @@ -7506,6 +7464,7 @@ dependencies = [ "re_chunk_store", "re_dataframe", "re_grpc_client", + "re_grpc_server", "re_log", "re_log_encoding", "re_log_types", @@ -7515,7 +7474,6 @@ dependencies = [ "re_sorbet", "re_video", "re_web_viewer_server", - "re_ws_comms", "tokio", "tokio-stream", "tonic", @@ -9232,27 +9190,6 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5902c5d130972a0000f60860bfbf46f7ca3db5391eddfedd1b8728bd9dc96c0e" -[[package]] -name = "tungstenite" -version = "0.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" -dependencies = [ - "byteorder", - "bytes", - "data-encoding", - "http 1.1.0", - "httparse", - "log", - "rand", - "rustls 0.23.18", - "rustls-pki-types", - "sha1", - "thiserror 1.0.65", - "utf-8", - "webpki-roots 0.26.6", -] - [[package]] name = "twox-hash" version = "1.6.3" @@ -9423,12 +9360,6 @@ dependencies = [ "tiny-skia-path", ] -[[package]] -name = "utf-8" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" - [[package]] name = "utf16_iter" version = "1.0.5" diff --git a/Cargo.toml b/Cargo.toml index b00ddc9147c8..414194d5eefd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,7 +64,6 @@ re_sdk_comms = { path = "crates/store/re_sdk_comms", version = "=0.22.0-alpha.1" re_sorbet = { path = "crates/store/re_sorbet", version = "=0.22.0-alpha.1", default-features = false } re_types = { path = "crates/store/re_types", version = "=0.22.0-alpha.1", default-features = false } re_types_core = { path = "crates/store/re_types_core", version = "=0.22.0-alpha.1", default-features = false } -re_ws_comms = { path = "crates/store/re_ws_comms", version = "=0.22.0-alpha.1", default-features = false } # crates/top: re_sdk = { path = "crates/top/re_sdk", version = "=0.22.0-alpha.1", default-features = false } diff --git a/crates/store/re_data_source/src/web_sockets.rs b/crates/store/re_data_source/src/web_sockets.rs deleted file mode 100644 index c258aa8e76ac..000000000000 --- a/crates/store/re_data_source/src/web_sockets.rs +++ /dev/null @@ -1,43 +0,0 @@ -use re_log_types::LogMsg; -use re_smart_channel::Receiver; - -/// `on_msg` can be used to wake up the UI thread on Wasm. -pub fn connect_to_ws_url( - url: &str, - on_msg: Option>, -) -> anyhow::Result> { - let (tx, rx) = re_smart_channel::smart_channel( - re_smart_channel::SmartMessageSource::WsClient { - ws_server_url: url.to_owned(), - }, - re_smart_channel::SmartChannelSource::WsClient { - ws_server_url: url.to_owned(), - }, - ); - - re_log::info!("Connecting to WebSocket server at {url:?}…"); - - let callback = { - let url = url.to_owned(); - move |binary: Vec| match re_ws_comms::decode_log_msg(&binary) { - Ok(log_msg) => { - if tx.send(log_msg).is_ok() { - if let Some(on_msg) = &on_msg { - on_msg(); - } - std::ops::ControlFlow::Continue(()) - } else { - re_log::info_once!("Closing connection to {url}"); - std::ops::ControlFlow::Break(()) - } - } - Err(err) => { - re_log::error!("Failed to parse message: {err}"); - std::ops::ControlFlow::Break(()) - } - } - }; - - re_ws_comms::viewer_to_server(url.to_owned(), callback)?; - Ok(rx) -} diff --git a/crates/store/re_ws_comms/Cargo.toml b/crates/store/re_ws_comms/Cargo.toml deleted file mode 100644 index 71d196783042..000000000000 --- a/crates/store/re_ws_comms/Cargo.toml +++ /dev/null @@ -1,61 +0,0 @@ -[package] -name = "re_ws_comms" -authors.workspace = true -description = "WebSocket communication library (encoding, decoding, client, server) between a Rerun server and viewer" -edition.workspace = true -homepage.workspace = true -include.workspace = true -license.workspace = true -publish = true -readme = "README.md" -repository.workspace = true -rust-version.workspace = true -version.workspace = true - -[lints] -workspace = true - -[package.metadata.docs.rs] -all-features = true - - -[features] -## Enable the client (viewer-side). -client = ["ewebsock"] - -## Enable the server. -server = [ - "dep:parking_lot", - "dep:re_smart_channel", - "dep:tungstenite", - "dep:polling", - "tungstenite/handshake", -] - -## Enable encryption using TLS support (`wss://`). -tls = [ - "ewebsock/tls", - "tungstenite/rustls-tls-webpki-roots", # TODO(emilk): there is some problem with this. check alternative tungstenite tls features -] - - -[dependencies] -re_format.workspace = true -re_log.workspace = true -re_log_types = { workspace = true, features = ["serde"] } -re_memory.workspace = true -re_tracing.workspace = true - -anyhow.workspace = true -bincode.workspace = true -document-features.workspace = true -thiserror.workspace = true - -# Client: -ewebsock = { workspace = true, optional = true } - -# Server: -parking_lot = { workspace = true, optional = true } -polling = { workspace = true, optional = true } -re_smart_channel = { workspace = true, optional = true } -tungstenite = { workspace = true, optional = true, default-features = false } diff --git a/crates/store/re_ws_comms/README.md b/crates/store/re_ws_comms/README.md deleted file mode 100644 index 9c6e1eac0ea9..000000000000 --- a/crates/store/re_ws_comms/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# re_ws_comms - -Part of the [`rerun`](https://github.com/rerun-io/rerun) family of crates. - -[![Latest version](https://img.shields.io/crates/v/re_ws_comms.svg)](https://crates.io/crates/re_ws_comms) -[![Documentation](https://docs.rs/re_ws_comms/badge.svg)](https://docs.rs/re_ws_comms) -![MIT](https://img.shields.io/badge/license-MIT-blue.svg) -![Apache](https://img.shields.io/badge/license-Apache-blue.svg) - -WebSocket communication library (encoding, decoding, client, server) between a Rerun server and viewer. diff --git a/crates/store/re_ws_comms/src/client.rs b/crates/store/re_ws_comms/src/client.rs deleted file mode 100644 index 6fa617f90023..000000000000 --- a/crates/store/re_ws_comms/src/client.rs +++ /dev/null @@ -1,60 +0,0 @@ -use std::ops::ControlFlow; - -use ewebsock::{WsEvent, WsMessage}; - -// TODO(jleibs): use thiserror -pub type Result = anyhow::Result; - -/// Connect viewer to server -pub fn viewer_to_server( - url: String, - on_binary_msg: impl Fn(Vec) -> ControlFlow<()> + Send + 'static, -) -> Result<()> { - let gigs = 1024 * 1024 * 1024; - let options = ewebsock::Options { - // This limits the size of one chunk of rerun log data when running a local websocket client. - // We set a very high limit, because we should be able to trust the server. - // See https://github.com/rerun-io/rerun/issues/5268 for more - max_incoming_frame_size: 2 * gigs, - ..ewebsock::Options::default() - }; - - ewebsock::ws_receive( - url.clone(), - options, - Box::new(move |event: WsEvent| match event { - WsEvent::Opened => { - re_log::info!("Connection to {url} established"); - ControlFlow::Continue(()) - } - WsEvent::Message(message) => match message { - WsMessage::Binary(binary) => on_binary_msg(binary), - WsMessage::Text(text) => { - re_log::warn!("Unexpected text message: {text:?}"); - ControlFlow::Continue(()) - } - WsMessage::Unknown(text) => { - re_log::warn!("Unknown message: {text:?}"); - ControlFlow::Continue(()) - } - WsMessage::Ping(_data) => { - re_log::warn!("Unexpected PING"); - ControlFlow::Continue(()) - } - WsMessage::Pong(_data) => { - re_log::warn!("Unexpected PONG"); - ControlFlow::Continue(()) - } - }, - WsEvent::Error(error) => { - re_log::error!("Connection error: {error}"); - ControlFlow::Break(()) - } - WsEvent::Closed => { - re_log::info!("Connection to {url} closed."); - ControlFlow::Break(()) - } - }), - ) - .map_err(|err| anyhow::format_err!("ewebsock: {err}")) -} diff --git a/crates/store/re_ws_comms/src/lib.rs b/crates/store/re_ws_comms/src/lib.rs deleted file mode 100644 index 6cdbacbb9d68..000000000000 --- a/crates/store/re_ws_comms/src/lib.rs +++ /dev/null @@ -1,112 +0,0 @@ -//! Communications between server and viewer. -//! -//! ## Feature flags -#![doc = document_features::document_features!()] -//! - -// TODO(#6330): remove unwrap() -#![allow(clippy::unwrap_used)] - -#[cfg(feature = "client")] -mod client; -use std::{fmt::Display, str::FromStr}; - -#[cfg(feature = "client")] -pub use client::viewer_to_server; - -#[cfg(feature = "server")] -mod server; -#[cfg(feature = "server")] -pub use server::RerunServer; - -use re_log_types::LogMsg; - -pub const DEFAULT_WS_SERVER_PORT: u16 = 9877; - -#[cfg(feature = "tls")] -pub const PROTOCOL: &str = "wss"; - -#[cfg(not(feature = "tls"))] -pub const PROTOCOL: &str = "ws"; - -// ---------------------------------------------------------------------------- - -/// Failure to host the Rerun WebSocket server. -#[derive(thiserror::Error, Debug)] -pub enum RerunServerError { - #[error("Failed to bind to WebSocket port {0}: {1}")] - BindFailed(RerunServerPort, std::io::Error), - - #[error("Failed to bind to WebSocket port {0} since the address is already in use. Use port 0 to let the OS choose a free port.")] - BindFailedAddrInUse(RerunServerPort), - - #[error("Received an invalid message")] - InvalidMessagePrefix, - - #[error("Received an invalid message")] - InvalidMessage(#[from] bincode::Error), - - #[cfg(feature = "server")] - #[error("IO error: {0}")] - IoError(#[from] std::io::Error), -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -/// Typed port for use with [`RerunServer`] -pub struct RerunServerPort(pub u16); - -impl Default for RerunServerPort { - fn default() -> Self { - Self(DEFAULT_WS_SERVER_PORT) - } -} - -impl Display for RerunServerPort { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.0) - } -} - -// Needed for clap -impl FromStr for RerunServerPort { - type Err = String; - - fn from_str(s: &str) -> Result { - match s.parse::() { - Ok(port) => Ok(Self(port)), - Err(err) => Err(format!("Failed to parse port: {err}")), - } - } -} - -/// Add a protocol (`ws://` or `wss://`) to the given address. -pub fn server_url(local_addr: &std::net::SocketAddr) -> String { - if local_addr.ip().is_unspecified() { - // "0.0.0.0" - format!("{PROTOCOL}://localhost:{}", local_addr.port()) - } else { - format!("{PROTOCOL}://{local_addr}") - } -} - -const PREFIX: [u8; 4] = *b"RR00"; - -pub fn encode_log_msg(log_msg: &LogMsg) -> Vec { - re_tracing::profile_function!(); - use bincode::Options as _; - let mut bytes = PREFIX.to_vec(); - bincode::DefaultOptions::new() - .serialize_into(&mut bytes, log_msg) - .unwrap(); - bytes -} - -pub fn decode_log_msg(data: &[u8]) -> Result { - re_tracing::profile_function!(); - let payload = data - .strip_prefix(&PREFIX) - .ok_or(RerunServerError::InvalidMessagePrefix)?; - - use bincode::Options as _; - Ok(bincode::DefaultOptions::new().deserialize(payload)?) -} diff --git a/crates/store/re_ws_comms/src/server.rs b/crates/store/re_ws_comms/src/server.rs deleted file mode 100644 index af9bbfeb0021..000000000000 --- a/crates/store/re_ws_comms/src/server.rs +++ /dev/null @@ -1,444 +0,0 @@ -//! The server is a pub-sub architecture. -//! -//! Each incoming log message is stored, and sent to any connected client. -//! Each connecting client is first sent the history of stored log messages. -//! -//! In the future thing will be changed to a protocol where the clients can query -//! for specific data based on e.g. time. - -use std::{ - collections::VecDeque, - net::{TcpListener, TcpStream}, - sync::{ - atomic::{AtomicBool, AtomicU64, Ordering}, - Arc, - }, -}; - -use parking_lot::Mutex; -use polling::{Event, Poller}; -use tungstenite::WebSocket; - -use re_log_types::LogMsg; -use re_memory::MemoryLimit; -use re_smart_channel::ReceiveSet; - -use crate::{server_url, RerunServerError, RerunServerPort}; - -struct MessageQueue { - server_memory_limit: MemoryLimit, - messages: VecDeque>, - - /// Never garbage collected. - messages_static: VecDeque>, -} - -impl MessageQueue { - pub fn new(server_memory_limit: MemoryLimit) -> Self { - Self { - server_memory_limit, - messages: Default::default(), - messages_static: Default::default(), - } - } - - pub fn push(&mut self, msg: Vec) { - self.gc_if_using_too_much_ram(); - self.messages.push_back(msg); - } - - /// Messages pushed using this method will stay around indefinitely. - /// - /// Useful e.g. for `SetStoreInfo` messages, so that clients late to the party actually get a - /// chance of receiving them. - pub fn push_static(&mut self, msg: Vec) { - self.gc_if_using_too_much_ram(); - self.messages_static.push_back(msg); - } - - fn gc_if_using_too_much_ram(&mut self) { - re_tracing::profile_function!(); - - if let Some(max_bytes) = self.server_memory_limit.max_bytes { - let max_bytes = max_bytes as u64; - let bytes_used = self.messages.iter().map(|m| m.len() as u64).sum::(); - - if max_bytes < bytes_used { - re_tracing::profile_scope!("Drop messages"); - re_log::info_once!( - "Memory limit ({}) exceeded. Dropping old log messages from the server. Clients connecting after this will not see the full history.", - re_format::format_bytes(max_bytes as _) - ); - - let bytes_to_free = bytes_used - max_bytes; - - let mut bytes_dropped = 0; - let mut messages_dropped = 0; - - while bytes_dropped < bytes_to_free { - if let Some(msg) = self.messages.pop_front() { - bytes_dropped += msg.len() as u64; - messages_dropped += 1; - } else { - break; - } - } - - re_log::trace!( - "Dropped {} bytes in {messages_dropped} message(s)", - re_format::format_bytes(bytes_dropped as _) - ); - } - } - } -} - -/// Websocket host for relaying [`LogMsg`]s to a web viewer. -/// -/// When dropped, the server will be shut down. -pub struct RerunServer { - local_addr: std::net::SocketAddr, - - listener_join_handle: Option>, - poller: Arc, - shutdown_flag: Arc, - - /// Total count; never decreasing. - num_accepted_clients: Arc, -} - -impl RerunServer { - /// Create new [`RerunServer`] to relay [`LogMsg`]s to a websocket. - /// The websocket will be available at `port`. - /// - /// A `bind_ip` of `"0.0.0.0"` is a good default. - /// A port of 0 will let the OS choose a free port. - /// - /// Once created, the server will immediately start listening for connections. - pub fn new( - rerun_rx: ReceiveSet, - bind_ip: &str, - port: RerunServerPort, - server_memory_limit: MemoryLimit, - ) -> Result { - let bind_addr = format!("{bind_ip}:{port}"); - - let listener_socket = TcpListener::bind(bind_addr).map_err(|err| { - if err.kind() == std::io::ErrorKind::AddrInUse { - RerunServerError::BindFailedAddrInUse(port) - } else { - RerunServerError::BindFailed(port, err) - } - })?; - - // Blocking listener socket seems much easier at first glance: - // No polling needed and as such no extra libraries! - // However, there is no portable way of stopping an `accept` call on a blocking socket. - // Therefore, we do the "correct thing" and use a non-blocking socket together with the `polling` library. - listener_socket.set_nonblocking(true)?; - - let poller = Arc::new(Poller::new()?); - let shutdown_flag = Arc::new(AtomicBool::new(false)); - let num_accepted_clients = Arc::new(AtomicU64::new(0)); - - let local_addr = listener_socket.local_addr()?; - let poller_copy = poller.clone(); - let shutdown_flag_copy = shutdown_flag.clone(); - let num_clients_copy = num_accepted_clients.clone(); - - let listener_join_handle = std::thread::Builder::new() - .name("rerun_ws_server: listener".to_owned()) - .spawn(move || { - Self::listen_thread_func( - &poller, - &listener_socket, - &ReceiveSetBroadcaster::new(rerun_rx, server_memory_limit), - &shutdown_flag, - &num_accepted_clients, - ); - })?; - - let slf = Self { - local_addr, - poller: poller_copy, - listener_join_handle: Some(listener_join_handle), - shutdown_flag: shutdown_flag_copy, - num_accepted_clients: num_clients_copy, - }; - - re_log::info!( - "Hosting a WebSocket server on {wsurl}. You can connect to this with a native viewer (`rerun {wsurl}`) or the web viewer (with `?url={wsurl}`).", - wsurl=slf.server_url() - ); - - Ok(slf) - } - - /// Contains the `ws://` or `wss://` prefix. - pub fn server_url(&self) -> String { - server_url(&self.local_addr) - } - - /// Total count; never decreasing. - pub fn num_accepted_clients(&self) -> u64 { - self.num_accepted_clients.load(Ordering::Relaxed) - } - - /// Blocks execution as long as the server is running. - /// - /// There's no way of shutting the server down from the outside right now. - pub fn block(mut self) { - if let Some(listener_join_handle) = self.listener_join_handle.take() { - listener_join_handle.join().ok(); - } - } - - fn listen_thread_func( - poller: &Poller, - listener_socket: &TcpListener, - message_broadcaster: &ReceiveSetBroadcaster, - shutdown_flag: &AtomicBool, - num_accepted_clients: &AtomicU64, - ) { - // Each socket in `poll::Poller` needs a "name". - // Doesn't matter much what we're using here, as long as it's not used for something else - // on the same poller. - let listener_poll_key = 1; - - #[allow(unsafe_code)] - // SAFETY: `poller.add` requires a matching call to `poller.delete`, which we have below - if let Err(err) = unsafe { poller.add(listener_socket, Event::readable(listener_poll_key)) } - { - re_log::error!("Error when polling listener socket for incoming connections: {err}"); - return; - } - - let mut events = polling::Events::new(); - loop { - if let Err(err) = poller.wait(&mut events, None) { - re_log::warn!("Error polling WebSocket server listener: {err}"); - } - - if shutdown_flag.load(std::sync::atomic::Ordering::Acquire) { - re_log::debug!("Stopping WebSocket server listener."); - break; - } - - for event in events.iter() { - if event.key == listener_poll_key { - Self::accept_connection( - listener_socket, - message_broadcaster, - poller, - listener_poll_key, - num_accepted_clients, - ); - } - } - events.clear(); - } - - // This MUST be called before dropping `poller`! - poller.delete(listener_socket).ok(); - } - - fn accept_connection( - listener_socket: &TcpListener, - message_broadcaster: &ReceiveSetBroadcaster, - poller: &Poller, - listener_poll_key: usize, - num_accepted_clients: &AtomicU64, - ) { - match listener_socket.accept() { - Ok((tcp_stream, _)) => { - let address = tcp_stream.peer_addr(); - - // Keep the client simple, otherwise we need to do polling there as well. - tcp_stream.set_nonblocking(false).ok(); - - re_log::debug!("New WebSocket connection from {address:?}"); - - match tungstenite::accept(tcp_stream) { - Ok(ws_stream) => { - message_broadcaster.add_client(ws_stream); - num_accepted_clients.fetch_add(1, Ordering::Relaxed); - } - Err(err) => { - re_log::warn!("Error accepting WebSocket connection: {err}"); - } - }; - } - - Err(err) => { - re_log::warn!("Error accepting WebSocket connection: {err}"); - } - }; - - // Set interest in the next readability event. - if let Err(err) = poller.modify(listener_socket, Event::readable(listener_poll_key)) { - re_log::error!("Error when polling listener socket for incoming connections: {err}"); - } - } - - fn stop_listener(&mut self) { - let Some(join_handle) = self.listener_join_handle.take() else { - return; - }; - - self.shutdown_flag - .store(true, std::sync::atomic::Ordering::Release); - - if let Err(err) = self.poller.notify() { - re_log::warn!("Error notifying WebSocket server listener: {err}"); - return; - } - - join_handle.join().ok(); - } -} - -impl Drop for RerunServer { - fn drop(&mut self) { - let num_accepted_clients = self.num_accepted_clients.load(Ordering::Relaxed); - re_log::info!( - "Shutting down Rerun server on {} after serving {num_accepted_clients} client(s)", - self.server_url() - ); - self.stop_listener(); - } -} - -/// Broadcasts messages to all connected clients and stores a history of messages to resend to new clients. -/// -/// This starts a thread which will close when the underlying `ReceiveSet` gets a quit message or looses all its connections. -struct ReceiveSetBroadcaster { - inner: Arc>, - shutdown_on_next_recv: Arc, -} - -/// Inner state of the [`ReceiveSetBroadcaster`], protected by a mutex. -struct ReceiveSetBroadcasterInnerState { - /// Don't allow adding to the history while adding/removing clients. - /// This way, no messages history is lost! - history: MessageQueue, - clients: Vec>, -} - -impl ReceiveSetBroadcaster { - pub fn new(log_rx: ReceiveSet, server_memory_limit: MemoryLimit) -> Self { - let inner = Arc::new(Mutex::new(ReceiveSetBroadcasterInnerState { - history: MessageQueue::new(server_memory_limit), - clients: Vec::new(), - })); - let shutdown = Arc::new(AtomicBool::new(false)); - - let inner_copy = inner.clone(); - let shutdown_copy = shutdown.clone(); - - if let Err(err) = std::thread::Builder::new() - .name("rerun_ws_server: broadcaster".to_owned()) - .spawn(move || { - Self::broadcast_thread_func(&log_rx, &inner, &shutdown); - }) - { - re_log::error!( - "Failed to spawn thread for broadcasting messages to websocket connections: {err}" - ); - } - - Self { - inner: inner_copy, - shutdown_on_next_recv: shutdown_copy, - } - } - - fn broadcast_thread_func( - log_rx: &ReceiveSet, - inner: &Mutex, - shutdown: &AtomicBool, - ) { - while let Ok(msg) = log_rx.recv() { - if shutdown.load(std::sync::atomic::Ordering::Acquire) { - re_log::debug!("Shutting down broadcaster."); - break; - } - - match msg.payload { - re_smart_channel::SmartMessagePayload::Msg(data) => { - let msg = crate::encode_log_msg(&data); - let mut inner = inner.lock(); - - // TODO(andreas): Should this be a parallel-for? - inner.clients.retain_mut(|client| { - if let Err(err) = client.send(tungstenite::Message::Binary(msg.clone())) { - re_log::warn!("Error sending message to web socket client: {err}"); - false - } else { - true - } - }); - - let msg_is_data = matches!(data, LogMsg::ArrowMsg(_, _)); - if msg_is_data { - inner.history.push(msg); - } else { - // Keep non-data commands around for clients late to the party. - inner.history.push_static(msg); - } - } - - re_smart_channel::SmartMessagePayload::Flush { on_flush_done } => { - on_flush_done(); - } - - re_smart_channel::SmartMessagePayload::Quit(err) => { - if let Some(err) = err { - re_log::warn!("Sender {} has left unexpectedly: {err}", msg.source); - } else { - re_log::debug!("Sender {} has left", msg.source); - } - } - } - - if log_rx.is_empty() { - re_log::debug!("No more connections. Shutting down broadcaster."); - break; - } - } - } - - /// Adds a websocket client to the broadcaster and replies all message history so far to it. - pub fn add_client(&self, mut client: WebSocket) { - // TODO(andreas): While it's great that we don't loose any messages while adding clients, - // the problem with this is that now we won't be able to keep the other clients fed, until this one is done! - // Meaning that if a new one connects, we stall the old connections until we have sent all messages to this one. - let mut inner = self.inner.lock(); - - for msg in &inner.history.messages_static { - if let Err(err) = client.send(tungstenite::Message::Binary(msg.clone())) { - re_log::warn!("Error sending static message to web socket client: {err}"); - return; - } - } - - for msg in &inner.history.messages { - if let Err(err) = client.send(tungstenite::Message::Binary(msg.clone())) { - re_log::warn!("Error sending message to web socket client: {err}"); - return; - } - } - - inner.clients.push(client); - } -} - -impl Drop for ReceiveSetBroadcaster { - fn drop(&mut self) { - // Close all connections and shut down the receive thread on the next message. - // This would only cause a dangling thread if the `ReceiveSet`'s channels are - // neither closing nor sending any more messages. - self.shutdown_on_next_recv - .store(true, std::sync::atomic::Ordering::Release); - self.inner.lock().clients.clear(); - } -} From 300e684648f5042298f872c66746635dd567eb70 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 29 Jan 2025 12:08:46 +0100 Subject: [PATCH 24/87] Make `MessageProxyAddress` public --- crates/store/re_grpc_client/src/message_proxy/mod.rs | 2 +- .../store/re_grpc_client/src/message_proxy/read.rs | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/crates/store/re_grpc_client/src/message_proxy/mod.rs b/crates/store/re_grpc_client/src/message_proxy/mod.rs index cd0bc3ca0535..6dcaba48a119 100644 --- a/crates/store/re_grpc_client/src/message_proxy/mod.rs +++ b/crates/store/re_grpc_client/src/message_proxy/mod.rs @@ -1,5 +1,5 @@ pub mod read; -pub use read::stream; +pub use read::{stream, MessageProxyAddress}; #[cfg(not(target_arch = "wasm32"))] pub mod write; diff --git a/crates/store/re_grpc_client/src/message_proxy/read.rs b/crates/store/re_grpc_client/src/message_proxy/read.rs index ce8ef23de4bf..a3b8da63ee98 100644 --- a/crates/store/re_grpc_client/src/message_proxy/read.rs +++ b/crates/store/re_grpc_client/src/message_proxy/read.rs @@ -33,10 +33,10 @@ pub fn stream( Ok(rx) } -struct MessageProxyAddress(String); +pub struct MessageProxyAddress(String); impl MessageProxyAddress { - fn parse(url: &str) -> Result { + pub fn parse(url: &str) -> Result { let Some(url) = url.strip_prefix("temp") else { let scheme = url.split_once("://").map(|(a, _)| a).ok_or("unknown"); return Err(InvalidMessageProxyAddress { @@ -69,6 +69,14 @@ impl Display for MessageProxyAddress { } } +impl std::str::FromStr for MessageProxyAddress { + type Err = InvalidMessageProxyAddress; + + fn from_str(s: &str) -> Result { + Self::parse(s) + } +} + #[derive(Debug, thiserror::Error)] #[error("invalid message proxy address {url:?}: {msg}")] pub struct InvalidMessageProxyAddress { From c83cc7ed74608cf60aa06cf80c82e11c6f1d8bf0 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 29 Jan 2025 12:09:21 +0100 Subject: [PATCH 25/87] Remove `re_sdk_comms` usage from `re_sdk` --- crates/top/re_sdk/Cargo.toml | 1 - crates/top/re_sdk/src/lib.rs | 7 +- crates/top/re_sdk/src/log_sink.rs | 38 ---- crates/top/re_sdk/src/recording_stream.rs | 249 +--------------------- crates/top/re_sdk/src/spawn.rs | 2 +- 5 files changed, 5 insertions(+), 292 deletions(-) diff --git a/crates/top/re_sdk/Cargo.toml b/crates/top/re_sdk/Cargo.toml index 4705f8205bba..4024ef46f239 100644 --- a/crates/top/re_sdk/Cargo.toml +++ b/crates/top/re_sdk/Cargo.toml @@ -59,7 +59,6 @@ re_log_encoding = { workspace = true, features = ["encoder"] } re_log_types.workspace = true re_log.workspace = true re_memory.workspace = true -re_sdk_comms = { workspace = true, features = ["client"] } re_types_core.workspace = true ahash.workspace = true diff --git a/crates/top/re_sdk/src/lib.rs b/crates/top/re_sdk/src/lib.rs index 340131b916da..ebee8848540b 100644 --- a/crates/top/re_sdk/src/lib.rs +++ b/crates/top/re_sdk/src/lib.rs @@ -33,8 +33,6 @@ pub use self::recording_stream::{ RecordingStreamResult, }; -pub use re_sdk_comms::{default_flush_timeout, default_server_addr}; - pub use re_log_types::{ entity_path, ApplicationId, EntityPath, EntityPathPart, Instance, StoreId, StoreKind, }; @@ -66,9 +64,7 @@ pub mod sink { pub use crate::binary_stream_sink::{ BinaryStreamSink, BinaryStreamSinkError, BinaryStreamStorage, }; - pub use crate::log_sink::{ - BufferedSink, CallbackSink, LogSink, MemorySink, MemorySinkStorage, TcpSink, - }; + pub use crate::log_sink::{BufferedSink, CallbackSink, LogSink, MemorySink, MemorySinkStorage}; pub use crate::log_sink::GrpcSink; @@ -115,7 +111,6 @@ pub mod external { pub use re_log; pub use re_log_encoding; pub use re_log_types; - pub use re_sdk_comms; pub use re_chunk::external::*; pub use re_log::external::*; diff --git a/crates/top/re_sdk/src/log_sink.rs b/crates/top/re_sdk/src/log_sink.rs index 7e61c7628c43..f76c37eddaed 100644 --- a/crates/top/re_sdk/src/log_sink.rs +++ b/crates/top/re_sdk/src/log_sink.rs @@ -331,44 +331,6 @@ impl LogSink for CallbackSink { // ---------------------------------------------------------------------------- -/// Stream log messages to a Rerun TCP server. -#[derive(Debug)] -pub struct TcpSink { - client: re_sdk_comms::Client, -} - -impl TcpSink { - /// Connect to the given address in a background thread. - /// Retries until successful. - /// - /// `flush_timeout` is the minimum time the [`TcpSink`] will wait during a flush - /// before potentially dropping data. Note: Passing `None` here can cause a - /// call to `flush` to block indefinitely if a connection cannot be established. - #[inline] - pub fn new(addr: std::net::SocketAddr, flush_timeout: Option) -> Self { - Self { - client: re_sdk_comms::Client::new(addr, flush_timeout), - } - } -} - -impl LogSink for TcpSink { - #[inline] - fn send(&self, msg: LogMsg) { - self.client.send(msg); - } - - #[inline] - fn flush_blocking(&self) { - self.client.flush(); - } - - #[inline] - fn drop_if_disconnected(&self) { - self.client.drop_if_disconnected(); - } -} - /// Stream log messages to an in-memory storage node. pub struct GrpcSink { client: MessageProxyClient, diff --git a/crates/top/re_sdk/src/recording_stream.rs b/crates/top/re_sdk/src/recording_stream.rs index a3e61c4d83bc..d2601f208df1 100644 --- a/crates/top/re_sdk/src/recording_stream.rs +++ b/crates/top/re_sdk/src/recording_stream.rs @@ -297,92 +297,6 @@ impl RecordingStreamBuilder { Ok((rec, storage)) } - /// Creates a new [`RecordingStream`] that is pre-configured to stream the data through to a - /// remote Rerun instance. - /// - /// See also [`Self::connect_opts`] if you wish to configure the TCP connection. - /// - /// ## Example - /// - /// ```no_run - /// let rec = re_sdk::RecordingStreamBuilder::new("rerun_example_app").connect()?; - /// # Ok::<(), Box>(()) - /// ``` - #[deprecated(since = "0.20.0", note = "use connect_tcp() instead")] - pub fn connect(self) -> RecordingStreamResult { - self.connect_tcp() - } - - /// Creates a new [`RecordingStream`] that is pre-configured to stream the data through to a - /// remote Rerun instance. - /// - /// See also [`Self::connect_opts`] if you wish to configure the TCP connection. - /// - /// ## Example - /// - /// ```no_run - /// let rec = re_sdk::RecordingStreamBuilder::new("rerun_example_app").connect_tcp()?; - /// # Ok::<(), Box>(()) - /// ``` - pub fn connect_tcp(self) -> RecordingStreamResult { - self.connect_tcp_opts(crate::default_server_addr(), crate::default_flush_timeout()) - } - - /// Creates a new [`RecordingStream`] that is pre-configured to stream the data through to a - /// remote Rerun instance. - /// - /// `flush_timeout` is the minimum time the [`TcpSink`][`crate::log_sink::TcpSink`] will - /// wait during a flush before potentially dropping data. Note: Passing `None` here can cause a - /// call to `flush` to block indefinitely if a connection cannot be established. - /// - /// ## Example - /// - /// ```no_run - /// let rec = re_sdk::RecordingStreamBuilder::new("rerun_example_app") - /// .connect_opts(re_sdk::default_server_addr(), re_sdk::default_flush_timeout())?; - /// # Ok::<(), Box>(()) - /// ``` - #[deprecated(since = "0.20.0", note = "use connect_tcp_opts() instead")] - pub fn connect_opts( - self, - addr: std::net::SocketAddr, - flush_timeout: Option, - ) -> RecordingStreamResult { - self.connect_tcp_opts(addr, flush_timeout) - } - - /// Creates a new [`RecordingStream`] that is pre-configured to stream the data through to a - /// remote Rerun instance. - /// - /// `flush_timeout` is the minimum time the [`TcpSink`][`crate::log_sink::TcpSink`] will - /// wait during a flush before potentially dropping data. Note: Passing `None` here can cause a - /// call to `flush` to block indefinitely if a connection cannot be established. - /// - /// ## Example - /// - /// ```no_run - /// let rec = re_sdk::RecordingStreamBuilder::new("rerun_example_app") - /// .connect_opts(re_sdk::default_server_addr(), re_sdk::default_flush_timeout())?; - /// # Ok::<(), Box>(()) - /// ``` - pub fn connect_tcp_opts( - self, - addr: std::net::SocketAddr, - flush_timeout: Option, - ) -> RecordingStreamResult { - let (enabled, store_info, batcher_config) = self.into_args(); - if enabled { - RecordingStream::new( - store_info, - batcher_config, - Box::new(crate::log_sink::TcpSink::new(addr, flush_timeout)), - ) - } else { - re_log::debug!("Rerun disabled - call to connect() ignored"); - Ok(RecordingStream::disabled()) - } - } - /// Creates a new [`RecordingStream`] that is pre-configured to stream the data through to a /// remote Rerun instance. /// @@ -492,68 +406,6 @@ impl RecordingStreamBuilder { } } - /// Spawns a new Rerun Viewer process from an executable available in PATH, then creates a new - /// [`RecordingStream`] that is pre-configured to stream the data through to that viewer over TCP. - /// - /// If a Rerun Viewer is already listening on this TCP port, the stream will be redirected to - /// that viewer instead of starting a new one. - /// - /// See also [`Self::spawn_opts`] if you wish to configure the behavior of thew Rerun process - /// as well as the underlying TCP connection. - /// - /// ## Example - /// - /// ```no_run - /// let rec = re_sdk::RecordingStreamBuilder::new("rerun_example_app").spawn()?; - /// # Ok::<(), Box>(()) - /// ``` - pub fn spawn(self) -> RecordingStreamResult { - self.spawn_opts(&Default::default(), crate::default_flush_timeout()) - } - - /// Spawns a new Rerun Viewer process from an executable available in PATH, then creates a new - /// [`RecordingStream`] that is pre-configured to stream the data through to that viewer over TCP. - /// - /// If a Rerun Viewer is already listening on this TCP port, the stream will be redirected to - /// that viewer instead of starting a new one. - /// - /// The behavior of the spawned Viewer can be configured via `opts`. - /// If you're fine with the default behavior, refer to the simpler [`Self::spawn`]. - /// - /// `flush_timeout` is the minimum time the [`TcpSink`][`crate::log_sink::TcpSink`] will - /// wait during a flush before potentially dropping data. Note: Passing `None` here can cause a - /// call to `flush` to block indefinitely if a connection cannot be established. - /// - /// ## Example - /// - /// ```no_run - /// let rec = re_sdk::RecordingStreamBuilder::new("rerun_example_app") - /// .spawn_opts(&re_sdk::SpawnOptions::default(), re_sdk::default_flush_timeout())?; - /// # Ok::<(), Box>(()) - /// ``` - pub fn spawn_opts( - self, - opts: &crate::SpawnOptions, - flush_timeout: Option, - ) -> RecordingStreamResult { - if !self.is_enabled() { - re_log::debug!("Rerun disabled - call to spawn() ignored"); - return Ok(RecordingStream::disabled()); - } - - let connect_addr = opts.connect_addr(); - - // NOTE: If `_RERUN_TEST_FORCE_SAVE` is set, all recording streams will write to disk no matter - // what, thus spawning a viewer is pointless (and probably not intended). - if forced_sink_path().is_some() { - return self.connect_tcp_opts(connect_addr, flush_timeout); - } - - crate::spawn(opts)?; - - self.connect_tcp_opts(connect_addr, flush_timeout) - } - /// Spawns a new Rerun Viewer process from an executable available in PATH, then creates a new /// [`RecordingStream`] that is pre-configured to stream the data through to that viewer over TCP. /// @@ -569,8 +421,8 @@ impl RecordingStreamBuilder { /// let rec = re_sdk::RecordingStreamBuilder::new("rerun_example_app").spawn_grpc()?; /// # Ok::<(), Box>(()) /// ``` - pub fn spawn_grpc(self) -> RecordingStreamResult { - self.spawn_grpc_opts(&Default::default()) + pub fn spawn(self) -> RecordingStreamResult { + self.spawn_opts(&Default::default()) } /// Spawns a new Rerun Viewer process from an executable available in PATH, then creates a new @@ -589,10 +441,7 @@ impl RecordingStreamBuilder { /// .spawn_grpc_opts(&re_sdk::SpawnOptions::default())?; /// # Ok::<(), Box>(()) /// ``` - pub fn spawn_grpc_opts( - self, - opts: &crate::SpawnOptions, - ) -> RecordingStreamResult { + pub fn spawn_opts(self, opts: &crate::SpawnOptions) -> RecordingStreamResult { if !self.is_enabled() { re_log::debug!("Rerun disabled - call to spawn() ignored"); return Ok(RecordingStream::disabled()); @@ -1781,43 +1630,6 @@ impl RecordingStream { } impl RecordingStream { - /// Swaps the underlying sink for a [`crate::log_sink::TcpSink`] sink pre-configured to use - /// the specified address. - /// - /// See also [`Self::connect_opts`] if you wish to configure the TCP connection. - /// - /// This is a convenience wrapper for [`Self::set_sink`] that upholds the same guarantees in - /// terms of data durability and ordering. - /// See [`Self::set_sink`] for more information. - pub fn connect(&self) { - self.connect_opts(crate::default_server_addr(), crate::default_flush_timeout()); - } - - /// Swaps the underlying sink for a [`crate::log_sink::TcpSink`] sink pre-configured to use - /// the specified address. - /// - /// `flush_timeout` is the minimum time the [`TcpSink`][`crate::log_sink::TcpSink`] will - /// wait during a flush before potentially dropping data. Note: Passing `None` here can cause a - /// call to `flush` to block indefinitely if a connection cannot be established. - /// - /// This is a convenience wrapper for [`Self::set_sink`] that upholds the same guarantees in - /// terms of data durability and ordering. - /// See [`Self::set_sink`] for more information. - pub fn connect_opts( - &self, - addr: std::net::SocketAddr, - flush_timeout: Option, - ) { - if forced_sink_path().is_some() { - re_log::debug!("Ignored setting new TcpSink since {ENV_FORCE_SAVE} is set"); - return; - } - - let sink = crate::log_sink::TcpSink::new(addr, flush_timeout); - - self.set_sink(Box::new(sink)); - } - /// Swaps the underlying sink for a [`crate::log_sink::GrpcSink`] sink pre-configured to use /// the specified address. /// @@ -1850,61 +1662,6 @@ impl RecordingStream { self.set_sink(Box::new(sink)); } - /// Spawns a new Rerun Viewer process from an executable available in PATH, then swaps the - /// underlying sink for a [`crate::log_sink::TcpSink`] sink pre-configured to send data to that - /// new process. - /// - /// If a Rerun Viewer is already listening on this TCP port, the stream will be redirected to - /// that viewer instead of starting a new one. - /// - /// See also [`Self::spawn_opts`] if you wish to configure the behavior of thew Rerun process - /// as well as the underlying TCP connection. - /// - /// This is a convenience wrapper for [`Self::set_sink`] that upholds the same guarantees in - /// terms of data durability and ordering. - /// See [`Self::set_sink`] for more information. - pub fn spawn(&self) -> RecordingStreamResult<()> { - self.spawn_opts(&Default::default(), crate::default_flush_timeout()) - } - - /// Spawns a new Rerun Viewer process from an executable available in PATH, then swaps the - /// underlying sink for a [`crate::log_sink::TcpSink`] sink pre-configured to send data to that - /// new process. - /// - /// If a Rerun Viewer is already listening on this TCP port, the stream will be redirected to - /// that viewer instead of starting a new one. - /// - /// The behavior of the spawned Viewer can be configured via `opts`. - /// If you're fine with the default behavior, refer to the simpler [`Self::spawn`]. - /// - /// `flush_timeout` is the minimum time the [`TcpSink`][`crate::log_sink::TcpSink`] will - /// wait during a flush before potentially dropping data. Note: Passing `None` here can cause a - /// call to `flush` to block indefinitely if a connection cannot be established. - /// - /// This is a convenience wrapper for [`Self::set_sink`] that upholds the same guarantees in - /// terms of data durability and ordering. - /// See [`Self::set_sink`] for more information. - pub fn spawn_opts( - &self, - opts: &crate::SpawnOptions, - flush_timeout: Option, - ) -> RecordingStreamResult<()> { - if !self.is_enabled() { - re_log::debug!("Rerun disabled - call to spawn() ignored"); - return Ok(()); - } - if forced_sink_path().is_some() { - re_log::debug!("Ignored setting new TcpSink since {ENV_FORCE_SAVE} is set"); - return Ok(()); - } - - crate::spawn(opts)?; - - self.connect_opts(opts.connect_addr(), flush_timeout); - - Ok(()) - } - /// Spawns a new Rerun Viewer process from an executable available in PATH, then swaps the /// underlying sink for a [`crate::log_sink::GrpcSink`] sink pre-configured to send data to that /// new process. diff --git a/crates/top/re_sdk/src/spawn.rs b/crates/top/re_sdk/src/spawn.rs index 2ab2dbec39e3..af0da37adbd0 100644 --- a/crates/top/re_sdk/src/spawn.rs +++ b/crates/top/re_sdk/src/spawn.rs @@ -58,7 +58,7 @@ const RERUN_BINARY: &str = "rerun"; impl Default for SpawnOptions { fn default() -> Self { Self { - port: crate::default_server_addr().port(), + port: re_grpc_server::DEFAULT_SERVER_PORT, wait_for_bind: false, memory_limit: "75%".into(), executable_name: RERUN_BINARY.into(), From 7b0ab51140db4055e5dbd52784aeb57bfe8d4b78 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 29 Jan 2025 12:09:36 +0100 Subject: [PATCH 26/87] Add `FromStr` for `MemoryLimit` --- crates/utils/re_memory/src/memory_limit.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/utils/re_memory/src/memory_limit.rs b/crates/utils/re_memory/src/memory_limit.rs index c1940219bc78..e0c089fb07fc 100644 --- a/crates/utils/re_memory/src/memory_limit.rs +++ b/crates/utils/re_memory/src/memory_limit.rs @@ -93,3 +93,11 @@ impl MemoryLimit { None } } + +impl std::str::FromStr for MemoryLimit { + type Err = String; + + fn from_str(s: &str) -> Result { + Self::parse(s) + } +} From e21373784f8b322a7fb5e3de7c7df3c7aaeba930 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 29 Jan 2025 12:10:00 +0100 Subject: [PATCH 27/87] Update `rerun` entrypoint/command parser --- crates/top/rerun/Cargo.toml | 2 +- crates/top/rerun/src/clap.rs | 24 ++++++++++++--------- crates/top/rerun/src/commands/entrypoint.rs | 8 +++---- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/crates/top/rerun/Cargo.toml b/crates/top/rerun/Cargo.toml index 55b86921111d..96453a49b7b1 100644 --- a/crates/top/rerun/Cargo.toml +++ b/crates/top/rerun/Cargo.toml @@ -103,7 +103,7 @@ run = [ "re_log_encoding/decoder", ] -## Support for running a TCP server that listens to incoming log messages from a Rerun SDK. +## Support for running a gRPC server that listens to incoming log messages from a Rerun SDK. server = ["dep:re_grpc_server"] ## Embed the Rerun SDK & built-in types and re-export all of their public symbols. diff --git a/crates/top/rerun/src/clap.rs b/crates/top/rerun/src/clap.rs index 47ea8d824b9b..d8a9966df2e6 100644 --- a/crates/top/rerun/src/clap.rs +++ b/crates/top/rerun/src/clap.rs @@ -1,6 +1,6 @@ //! Integration with integration with the [`clap`](https://crates.io/crates/clap) command line argument parser. -use std::{net::SocketAddr, path::PathBuf}; +use std::path::PathBuf; use re_sdk::{RecordingStream, RecordingStreamBuilder}; @@ -8,7 +8,7 @@ use re_sdk::{RecordingStream, RecordingStreamBuilder}; #[derive(Debug, Clone, PartialEq, Eq)] enum RerunBehavior { - Connect(SocketAddr), + Connect(String), Save(PathBuf), @@ -55,10 +55,10 @@ pub struct RerunArgs { /// Connects and sends the logged data to a remote Rerun viewer. /// - /// Optionally takes an `ip:port`. + /// Optionally takes an HTTP(S) URL. #[clap(long)] #[allow(clippy::option_option)] - connect: Option>, + connect: Option>, /// Connects and sends the logged data to a web-based Rerun viewer. #[cfg(feature = "web_viewer")] @@ -110,9 +110,8 @@ impl RerunArgs { Default::default(), )), - RerunBehavior::Connect(addr) => Ok(( - RecordingStreamBuilder::new(application_id) - .connect_tcp_opts(addr, crate::default_flush_timeout())?, + RerunBehavior::Connect(url) => Ok(( + RecordingStreamBuilder::new(application_id).connect_grpc_opts(url)?, Default::default(), )), @@ -165,9 +164,14 @@ impl RerunArgs { return Ok(RerunBehavior::Serve); } - match self.connect { - Some(Some(addr)) => return Ok(RerunBehavior::Connect(addr)), - Some(None) => return Ok(RerunBehavior::Connect(crate::default_server_addr())), + match &self.connect { + Some(Some(url)) => return Ok(RerunBehavior::Connect(url.clone())), + Some(None) => { + return Ok(RerunBehavior::Connect(format!( + "http://127.0.0.1:{}", + re_grpc_server::DEFAULT_SERVER_PORT, + ))) + } None => {} } diff --git a/crates/top/rerun/src/commands/entrypoint.rs b/crates/top/rerun/src/commands/entrypoint.rs index 4356bdd46b91..7280f4a10110 100644 --- a/crates/top/rerun/src/commands/entrypoint.rs +++ b/crates/top/rerun/src/commands/entrypoint.rs @@ -702,14 +702,14 @@ fn run_impl( #[cfg(feature = "web_viewer")] if data_sources.len() == 1 && args.web_viewer { - if let DataSource::WebSocketAddr(rerun_server_ws_url) = data_sources[0].clone() { - // Special case! We are connecting a web-viewer to a web-socket address. - // Instead of piping, just host a web-viewer that connects to the web-socket directly: + if let DataSource::MessageProxy { url } = data_sources[0].clone() { + // Special case! We are connecting a web-viewer to a gRPC address. + // Instead of piping, just host a web-viewer that connects to the gRPC server directly: WebViewerConfig { bind_ip: args.bind.to_string(), web_port: args.web_viewer_port, - source_url: Some(rerun_server_ws_url), + source_url: Some(url), force_wgpu_backend: args.renderer, video_decoder: args.video_decoder, open_browser: true, From 74707f3a75f6c3aad2e7fabb057417b38af9f983 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 29 Jan 2025 12:10:11 +0100 Subject: [PATCH 28/87] Remove `re_sdk_comms` from `re_viewer` --- crates/viewer/re_viewer/Cargo.toml | 1 - crates/viewer/re_viewer/src/app.rs | 19 +------------------ 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/crates/viewer/re_viewer/Cargo.toml b/crates/viewer/re_viewer/Cargo.toml index 1176087a5185..3a19f420d52c 100644 --- a/crates/viewer/re_viewer/Cargo.toml +++ b/crates/viewer/re_viewer/Cargo.toml @@ -71,7 +71,6 @@ re_memory.workspace = true re_query.workspace = true re_renderer = { workspace = true, default-features = false } re_selection_panel.workspace = true -re_sdk_comms.workspace = true re_smart_channel.workspace = true re_view_bar_chart.workspace = true re_view_dataframe.workspace = true diff --git a/crates/viewer/re_viewer/src/app.rs b/crates/viewer/re_viewer/src/app.rs index 8392167e2bee..c44334375158 100644 --- a/crates/viewer/re_viewer/src/app.rs +++ b/crates/viewer/re_viewer/src/app.rs @@ -1222,24 +1222,7 @@ impl App { re_smart_channel::SmartMessagePayload::Quit(err) => { if let Some(err) = err { - let log_msg = - format!("Data source {} has left unexpectedly: {err}", msg.source); - - #[cfg(not(target_arch = "wasm32"))] - if err - .downcast_ref::() - .is_some_and(|e| { - matches!(e, re_sdk_comms::ConnectionError::UnknownClient) - }) - { - // This can happen if a client tried to connect but didn't send the `re_sdk_comms::PROTOCOL_HEADER`. - // Likely an unknown client stumbled onto the wrong port - don't log as an error. - // (for more information see https://github.com/rerun-io/rerun/issues/5883). - re_log::debug!("{log_msg}"); - continue; - } - - re_log::warn!("{log_msg}"); + re_log::warn!("Data source {} has left unexpectedly: {err}", msg.source); } else { re_log::debug!("Data source {} has finished", msg.source); } From 352a674de2636da8254eabf4f011b66e5762f381 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 29 Jan 2025 12:16:31 +0100 Subject: [PATCH 29/87] Remove `re_sdk_comms` from C/C++ --- crates/top/rerun_c/src/lib.rs | 79 ------------------- .../quick_start_connect.cpp | 2 +- .../all/concepts/app-model/native-sync.cpp | 2 +- .../all/quick_start/quick_start_connect.cpp | 2 +- examples/cpp/log_file/main.cpp | 2 +- rerun_cpp/docs/readme_snippets.cpp | 2 +- rerun_cpp/src/rerun/c/rerun.h | 39 +-------- rerun_cpp/src/rerun/recording_stream.cpp | 29 ------- rerun_cpp/tests/recording_stream.cpp | 4 +- tests/cpp/plot_dashboard_stress/main.cpp | 2 +- 10 files changed, 9 insertions(+), 154 deletions(-) diff --git a/crates/top/rerun_c/src/lib.rs b/crates/top/rerun_c/src/lib.rs index 264d75ff486a..ea77168a6a4f 100644 --- a/crates/top/rerun_c/src/lib.rs +++ b/crates/top/rerun_c/src/lib.rs @@ -557,45 +557,6 @@ pub extern "C" fn rr_recording_stream_flush_blocking(id: CRecordingStream) { } } -#[allow(clippy::result_large_err)] -fn rr_recording_stream_connect_impl( - stream: CRecordingStream, - tcp_addr: CStringView, - flush_timeout_sec: f32, -) -> Result<(), CError> { - let stream = recording_stream(stream)?; - - let tcp_addr = tcp_addr.as_str("tcp_addr")?; - let tcp_addr = tcp_addr.parse().map_err(|err| { - CError::new( - CErrorCode::InvalidSocketAddress, - &format!("Failed to parse tcp address {tcp_addr:?}: {err}"), - ) - })?; - - let flush_timeout = if flush_timeout_sec >= 0.0 { - Some(std::time::Duration::from_secs_f32(flush_timeout_sec)) - } else { - None - }; - stream.connect_opts(tcp_addr, flush_timeout); - - Ok(()) -} - -#[allow(unsafe_code)] -#[no_mangle] -pub extern "C" fn rr_recording_stream_connect( - id: CRecordingStream, - tcp_addr: CStringView, - flush_timeout_sec: f32, - error: *mut CError, -) { - if let Err(err) = rr_recording_stream_connect_impl(id, tcp_addr, flush_timeout_sec) { - err.write_error(error); - } -} - #[allow(clippy::result_large_err)] fn rr_recording_stream_connect_grpc_impl( stream: CRecordingStream, @@ -622,46 +583,6 @@ pub extern "C" fn rr_recording_stream_connect_grpc( } } -#[allow(clippy::result_large_err)] -fn rr_recording_stream_spawn_impl( - stream: CRecordingStream, - spawn_opts: *const CSpawnOptions, - flush_timeout_sec: f32, -) -> Result<(), CError> { - let stream = recording_stream(stream)?; - - let spawn_opts = if spawn_opts.is_null() { - re_sdk::SpawnOptions::default() - } else { - let spawn_opts = ptr::try_ptr_as_ref(spawn_opts, "spawn_opts")?; - spawn_opts.as_rust()? - }; - let flush_timeout = if flush_timeout_sec >= 0.0 { - Some(std::time::Duration::from_secs_f32(flush_timeout_sec)) - } else { - None - }; - - stream - .spawn_opts(&spawn_opts, flush_timeout) - .map_err(|err| CError::new(CErrorCode::RecordingStreamSpawnFailure, &err.to_string()))?; - - Ok(()) -} - -#[allow(unsafe_code)] -#[no_mangle] -pub extern "C" fn rr_recording_stream_spawn( - id: CRecordingStream, - spawn_opts: *const CSpawnOptions, - flush_timeout_sec: f32, - error: *mut CError, -) { - if let Err(err) = rr_recording_stream_spawn_impl(id, spawn_opts, flush_timeout_sec) { - err.write_error(error); - } -} - #[allow(clippy::result_large_err)] fn rr_recording_stream_spawn_grpc_impl( stream: CRecordingStream, diff --git a/crates/viewer/re_viewer/data/quick_start_guides/quick_start_connect.cpp b/crates/viewer/re_viewer/data/quick_start_guides/quick_start_connect.cpp index 1bdd80ab984e..733c63c2aafa 100644 --- a/crates/viewer/re_viewer/data/quick_start_guides/quick_start_connect.cpp +++ b/crates/viewer/re_viewer/data/quick_start_guides/quick_start_connect.cpp @@ -6,7 +6,7 @@ using namespace rerun::demo; int main() { // Create a new `RecordingStream` which sends data over TCP to the viewer process. const auto rec = rerun::RecordingStream("rerun_example_quick_start_connect"); - rec.connect_tcp().exit_on_failure(); + rec.connect_grpc().exit_on_failure(); // Create some data using the `grid` utility function. std::vector points = grid3d(-10.f, 10.f, 10); diff --git a/docs/snippets/all/concepts/app-model/native-sync.cpp b/docs/snippets/all/concepts/app-model/native-sync.cpp index 1e28f93a0f69..3a63d5848bda 100644 --- a/docs/snippets/all/concepts/app-model/native-sync.cpp +++ b/docs/snippets/all/concepts/app-model/native-sync.cpp @@ -4,7 +4,7 @@ int main() { // Connect to the Rerun TCP server using the default address and // port: localhost:9876 const auto rec = rerun::RecordingStream("rerun_example_native_sync"); - rec.connect_tcp().exit_on_failure(); + rec.connect_grpc().exit_on_failure(); // Log data as usual, thereby pushing it into the TCP socket. while (true) { diff --git a/docs/snippets/all/quick_start/quick_start_connect.cpp b/docs/snippets/all/quick_start/quick_start_connect.cpp index 1bdd80ab984e..733c63c2aafa 100644 --- a/docs/snippets/all/quick_start/quick_start_connect.cpp +++ b/docs/snippets/all/quick_start/quick_start_connect.cpp @@ -6,7 +6,7 @@ using namespace rerun::demo; int main() { // Create a new `RecordingStream` which sends data over TCP to the viewer process. const auto rec = rerun::RecordingStream("rerun_example_quick_start_connect"); - rec.connect_tcp().exit_on_failure(); + rec.connect_grpc().exit_on_failure(); // Create some data using the `grid` utility function. std::vector points = grid3d(-10.f, 10.f, 10); diff --git a/examples/cpp/log_file/main.cpp b/examples/cpp/log_file/main.cpp index cedba41b387d..fac89914b51e 100644 --- a/examples/cpp/log_file/main.cpp +++ b/examples/cpp/log_file/main.cpp @@ -43,7 +43,7 @@ int main(int argc, char** argv) { if (args["spawn"].as()) { rec.spawn().exit_on_failure(); } else if (args["connect"].as()) { - rec.connect_tcp().exit_on_failure(); + rec.connect_grpc().exit_on_failure(); } else if (args["stdout"].as()) { rec.to_stdout().exit_on_failure(); } else if (args.count("save")) { diff --git a/rerun_cpp/docs/readme_snippets.cpp b/rerun_cpp/docs/readme_snippets.cpp index 4c55ef12418d..5d79c9276416 100644 --- a/rerun_cpp/docs/readme_snippets.cpp +++ b/rerun_cpp/docs/readme_snippets.cpp @@ -45,7 +45,7 @@ static std::vector create_image() { [[maybe_unused]] static void connecting() { /// [Connecting] rerun::RecordingStream rec("rerun_example_app"); - auto result = rec.connect_tcp(); // Connect to local host with default port. + auto result = rec.connect_grpc(); // Connect to local host with default port. if (result.is_err()) { // Handle error. } diff --git a/rerun_cpp/src/rerun/c/rerun.h b/rerun_cpp/src/rerun/c/rerun.h index ddf8ed000591..95884b67f6ae 100644 --- a/rerun_cpp/src/rerun/c/rerun.h +++ b/rerun_cpp/src/rerun/c/rerun.h @@ -412,22 +412,7 @@ extern void rr_recording_stream_set_thread_local( /// Check whether the recording stream is enabled. extern bool rr_recording_stream_is_enabled(rr_recording_stream stream, rr_error* error); -/// Connect to a remote Rerun Viewer on the given ip:port. -/// -/// Requires that you first start a Rerun Viewer by typing 'rerun' in a terminal. -/// -/// flush_timeout_sec: -/// The minimum time the SDK will wait during a flush before potentially -/// dropping data if progress is not being made. Passing a negative value indicates no timeout, -/// and can cause a call to `flush` to block indefinitely. -/// -/// This function returns immediately and will only raise an error for argument parsing errors, -/// not for connection errors as these happen asynchronously. -extern void rr_recording_stream_connect( - rr_recording_stream stream, rr_string tcp_addr, float flush_timeout_sec, rr_error* error -); - -/// Connect to a remote Rerun Viewer on the given URL. +/// Connect to a remote Rerun Viewer on the given HTTP(S) URL. /// /// Requires that you first start a Rerun Viewer by typing 'rerun' in a terminal. /// @@ -442,28 +427,6 @@ extern void rr_recording_stream_connect_grpc( rr_recording_stream stream, rr_string url, rr_error* error ); -/// Spawns a new Rerun Viewer process from an executable available in PATH, then connects to it -/// over TCP. -/// -/// This function returns immediately and will only raise an error for argument parsing errors, -/// not for connection errors as these happen asynchronously. -/// -/// ## Parameters -/// -/// spawn_opts: -/// Configuration of the spawned process. -/// Refer to `rr_spawn_options` documentation for details. -/// Passing null is valid and will result in the recommended defaults. -/// -/// flush_timeout_sec: -/// The minimum time the SDK will wait during a flush before potentially -/// dropping data if progress is not being made. Passing a negative value indicates no timeout, -/// and can cause a call to `flush` to block indefinitely. -extern void rr_recording_stream_spawn( - rr_recording_stream stream, const rr_spawn_options* spawn_opts, float flush_timeout_sec, - rr_error* error -); - /// Spawns a new Rerun Viewer process from an executable available in PATH, then connects to it /// over gRPC. /// diff --git a/rerun_cpp/src/rerun/recording_stream.cpp b/rerun_cpp/src/rerun/recording_stream.cpp index 0e04aba79462..4feac124deda 100644 --- a/rerun_cpp/src/rerun/recording_stream.cpp +++ b/rerun_cpp/src/rerun/recording_stream.cpp @@ -104,35 +104,6 @@ namespace rerun { } } - Error RecordingStream::connect(std::string_view tcp_addr, float flush_timeout_sec) const { - return RecordingStream::connect_tcp(tcp_addr, flush_timeout_sec); - } - - Error RecordingStream::connect_tcp(std::string_view tcp_addr, float flush_timeout_sec) const { - rr_error status = {}; - rr_recording_stream_connect( - _id, - detail::to_rr_string(tcp_addr), - flush_timeout_sec, - &status - ); - return status; - } - - Error RecordingStream::connect_grpc(std::string_view url) const { - rr_error status = {}; - rr_recording_stream_connect_grpc(_id, detail::to_rr_string(url), &status); - return status; - } - - Error RecordingStream::spawn(const SpawnOptions& options, float flush_timeout_sec) const { - rr_spawn_options rerun_c_options = {}; - options.fill_rerun_c_struct(rerun_c_options); - rr_error status = {}; - rr_recording_stream_spawn(_id, &rerun_c_options, flush_timeout_sec, &status); - return status; - } - Error RecordingStream::spawn_grpc(const SpawnOptions& options) const { rr_spawn_options rerun_c_options = {}; options.fill_rerun_c_struct(rerun_c_options); diff --git a/rerun_cpp/tests/recording_stream.cpp b/rerun_cpp/tests/recording_stream.cpp index c1e77db5ebf5..0dda46ca45c7 100644 --- a/rerun_cpp/tests/recording_stream.cpp +++ b/rerun_cpp/tests/recording_stream.cpp @@ -380,14 +380,14 @@ void test_logging_to_connection(const char* address, const rerun::RecordingStrea AND_GIVEN("an invalid address for the socket address") { THEN("then the save call fails") { CHECK( - stream.connect_tcp("definitely not valid!", 0.0f).code == + stream.connect_grpc("definitely not valid!").code == rerun::ErrorCode::InvalidSocketAddress ); } } AND_GIVEN("a valid socket address " << address) { THEN("save call with zero timeout returns no error") { - REQUIRE(stream.connect_tcp(address, 0.0f).is_ok()); + REQUIRE(stream.connect_grpc(address).is_ok()); WHEN("logging a component and then flushing") { check_logged_error([&] { diff --git a/tests/cpp/plot_dashboard_stress/main.cpp b/tests/cpp/plot_dashboard_stress/main.cpp index 17b7650f5aaa..11c2f3328ed0 100644 --- a/tests/cpp/plot_dashboard_stress/main.cpp +++ b/tests/cpp/plot_dashboard_stress/main.cpp @@ -59,7 +59,7 @@ int main(int argc, char** argv) { if (args["spawn"].as()) { rec.spawn().exit_on_failure(); } else if (args["connect"].as()) { - rec.connect_tcp().exit_on_failure(); + rec.connect_grpc().exit_on_failure(); } else if (args["stdout"].as()) { rec.to_stdout().exit_on_failure(); } else if (args.count("save")) { From 0a0ebf7b8edf60669c1de632f195532d354da0fe Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 29 Jan 2025 12:47:01 +0100 Subject: [PATCH 30/87] Expose url parse error --- crates/store/re_grpc_client/src/lib.rs | 1 + .../re_grpc_client/src/message_proxy/mod.rs | 2 +- .../re_grpc_client/src/message_proxy/read.rs | 63 ++++++++++++++----- .../re_grpc_client/src/message_proxy/write.rs | 8 +-- crates/top/re_sdk/src/log_sink.rs | 3 +- crates/top/re_sdk/src/recording_stream.rs | 25 +++++--- crates/top/rerun/src/commands/entrypoint.rs | 4 +- crates/top/rerun_c/src/lib.rs | 9 ++- rerun_cpp/src/rerun/error.hpp | 2 +- rerun_py/Cargo.toml | 4 +- rerun_py/src/python_bridge.rs | 18 ++++-- 11 files changed, 99 insertions(+), 40 deletions(-) diff --git a/crates/store/re_grpc_client/src/lib.rs b/crates/store/re_grpc_client/src/lib.rs index ce6eaf738c33..bca8d5776509 100644 --- a/crates/store/re_grpc_client/src/lib.rs +++ b/crates/store/re_grpc_client/src/lib.rs @@ -1,6 +1,7 @@ //! Communications with an Rerun Data Platform gRPC server. pub mod message_proxy; +pub use message_proxy::MessageProxyUrl; #[cfg(feature = "redap")] pub mod redap; diff --git a/crates/store/re_grpc_client/src/message_proxy/mod.rs b/crates/store/re_grpc_client/src/message_proxy/mod.rs index 6dcaba48a119..c55a8a6d9953 100644 --- a/crates/store/re_grpc_client/src/message_proxy/mod.rs +++ b/crates/store/re_grpc_client/src/message_proxy/mod.rs @@ -1,5 +1,5 @@ pub mod read; -pub use read::{stream, MessageProxyAddress}; +pub use read::{stream, MessageProxyUrl}; #[cfg(not(target_arch = "wasm32"))] pub mod write; diff --git a/crates/store/re_grpc_client/src/message_proxy/read.rs b/crates/store/re_grpc_client/src/message_proxy/read.rs index a3b8da63ee98..6deb558c355f 100644 --- a/crates/store/re_grpc_client/src/message_proxy/read.rs +++ b/crates/store/re_grpc_client/src/message_proxy/read.rs @@ -13,10 +13,10 @@ use crate::TonicStatusError; pub fn stream( url: &str, on_msg: Option>, -) -> Result, InvalidMessageProxyAddress> { +) -> Result, InvalidMessageProxyUrl> { re_log::debug!("Loading {url} via gRPC…"); - let parsed_url = MessageProxyAddress::parse(url)?; + let parsed_url = MessageProxyUrl::parse(url)?; let url = url.to_owned(); let (tx, rx) = re_smart_channel::smart_channel( @@ -33,13 +33,24 @@ pub fn stream( Ok(rx) } -pub struct MessageProxyAddress(String); +/// Represents a URL to a gRPC server. +pub struct MessageProxyUrl(String); + +impl MessageProxyUrl { + /// Parses as a regular URL, the protocol must be `temp://`, `http://`, or `https://`. + pub fn parse(url: &str) -> Result { + if let Some(url) = url.strip_prefix("http") { + let _ = Url::parse(url).map_err(|err| InvalidMessageProxyUrl { + url: url.to_owned(), + msg: err.to_string(), + })?; + + return Ok(Self(url.to_owned())); + } -impl MessageProxyAddress { - pub fn parse(url: &str) -> Result { let Some(url) = url.strip_prefix("temp") else { let scheme = url.split_once("://").map(|(a, _)| a).ok_or("unknown"); - return Err(InvalidMessageProxyAddress { + return Err(InvalidMessageProxyUrl { url: url.to_owned(), msg: format!( "Invalid scheme {scheme:?}, expected {:?}", @@ -50,7 +61,7 @@ impl MessageProxyAddress { }; let url = format!("http{url}"); - let _ = Url::parse(&url).map_err(|err| InvalidMessageProxyAddress { + let _ = Url::parse(&url).map_err(|err| InvalidMessageProxyUrl { url: url.clone(), msg: err.to_string(), })?; @@ -58,19 +69,19 @@ impl MessageProxyAddress { Ok(Self(url)) } - fn to_http(&self) -> String { + pub fn to_http(&self) -> String { self.0.clone() } } -impl Display for MessageProxyAddress { +impl Display for MessageProxyUrl { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { Display::fmt(&self.0, f) } } -impl std::str::FromStr for MessageProxyAddress { - type Err = InvalidMessageProxyAddress; +impl std::str::FromStr for MessageProxyUrl { + type Err = InvalidMessageProxyUrl; fn from_str(s: &str) -> Result { Self::parse(s) @@ -78,14 +89,14 @@ impl std::str::FromStr for MessageProxyAddress { } #[derive(Debug, thiserror::Error)] -#[error("invalid message proxy address {url:?}: {msg}")] -pub struct InvalidMessageProxyAddress { +#[error("invalid message proxy url {url:?}: {msg}")] +pub struct InvalidMessageProxyUrl { pub url: String, pub msg: String, } async fn stream_async( - url: MessageProxyAddress, + url: MessageProxyUrl, tx: &re_smart_channel::Sender, on_msg: Option>, ) -> Result<(), StreamError> { @@ -143,3 +154,27 @@ async fn stream_async( Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + macro_rules! test_parse_url { + ($name:ident, $url:literal, error) => { + #[test] + fn $name() { + assert!(MessageProxyUrl::parse($url).is_err()); + } + }; + + ($name:ident, $url:literal, expected: $expected_http:literal) => { + #[test] + fn $name() { + assert_eq!(MessageProxyUrl::parse($url).to_http(), $expected_http); + } + }; + } + + test_parse_url!(basic, "temp://127.0.0.1:1852", expected: "http://127.0.0.1:1852"); + test_parse_url!(invalid, "definitely not valid", error); +} diff --git a/crates/store/re_grpc_client/src/message_proxy/write.rs b/crates/store/re_grpc_client/src/message_proxy/write.rs index 512c41716b0a..5c23aad6725f 100644 --- a/crates/store/re_grpc_client/src/message_proxy/write.rs +++ b/crates/store/re_grpc_client/src/message_proxy/write.rs @@ -13,6 +13,8 @@ use tokio::sync::mpsc::UnboundedSender; use tokio::sync::oneshot; use tonic::transport::Endpoint; +use super::MessageProxyUrl; + enum Cmd { LogMsg(LogMsg), Flush(oneshot::Sender<()>), @@ -38,10 +40,8 @@ pub struct Client { } impl Client { - /// `url` should be the `http://` or `https://` URL of the server. #[expect(clippy::needless_pass_by_value)] - pub fn new(url: impl Into, options: Options) -> Self { - let url: String = url.into(); + pub fn new(url: MessageProxyUrl, options: Options) -> Self { let (cmd_tx, cmd_rx) = mpsc::unbounded_channel(); let (shutdown_tx, shutdown_rx) = mpsc::channel(1); @@ -54,7 +54,7 @@ impl Client { .build() .expect("Failed to build tokio runtime") .block_on(message_proxy_client( - url, + url.to_http(), cmd_rx, shutdown_rx, options.compression, diff --git a/crates/top/re_sdk/src/log_sink.rs b/crates/top/re_sdk/src/log_sink.rs index f76c37eddaed..893af45e005a 100644 --- a/crates/top/re_sdk/src/log_sink.rs +++ b/crates/top/re_sdk/src/log_sink.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use parking_lot::Mutex; use re_grpc_client::message_proxy::write::Client as MessageProxyClient; +use re_grpc_client::message_proxy::MessageProxyUrl; use re_log_encoding::encoder::encode_as_bytes_local; use re_log_encoding::encoder::{local_raw_encoder, EncodeError}; use re_log_types::{BlueprintActivationCommand, LogMsg, StoreId}; @@ -345,7 +346,7 @@ impl GrpcSink { /// GrpcSink::new("http://127.0.0.1:9434"); /// ``` #[inline] - pub fn new(url: impl Into) -> Self { + pub fn new(url: MessageProxyUrl) -> Self { Self { client: MessageProxyClient::new(url, Default::default()), } diff --git a/crates/top/re_sdk/src/recording_stream.rs b/crates/top/re_sdk/src/recording_stream.rs index d2601f208df1..8cff390f4eaf 100644 --- a/crates/top/re_sdk/src/recording_stream.rs +++ b/crates/top/re_sdk/src/recording_stream.rs @@ -86,6 +86,10 @@ pub enum RecordingStreamError { #[cfg(feature = "data_loaders")] #[error(transparent)] DataLoaderError(#[from] re_data_loader::DataLoaderError), + + /// Invalid gRPC server address. + #[error(transparent)] + InvalidGrpcUrl(#[from] re_grpc_client::message_proxy::read::InvalidMessageProxyUrl), } /// Results that can occur when creating/manipulating a [`RecordingStream`]. @@ -334,7 +338,7 @@ impl RecordingStreamBuilder { RecordingStream::new( store_info, batcher_config, - Box::new(crate::log_sink::GrpcSink::new(url)), + Box::new(crate::log_sink::GrpcSink::new(url.into().parse()?)), ) } else { re_log::debug!("Rerun disabled - call to connect() ignored"); @@ -1639,10 +1643,11 @@ impl RecordingStream { /// terms of data durability and ordering. /// See [`Self::set_sink`] for more information. pub fn connect_grpc(&self) { - self.connect_grpc_opts(format!( - "http://127.0.0.1:{}", - re_grpc_server::DEFAULT_SERVER_PORT - )); + self.connect_grpc_opts( + format!("http://127.0.0.1:{}", re_grpc_server::DEFAULT_SERVER_PORT) + .parse() + .expect("should always be valid"), + ); } /// Swaps the underlying sink for a [`crate::log_sink::GrpcSink`] sink pre-configured to use @@ -1651,7 +1656,7 @@ impl RecordingStream { /// This is a convenience wrapper for [`Self::set_sink`] that upholds the same guarantees in /// terms of data durability and ordering. /// See [`Self::set_sink`] for more information. - pub fn connect_grpc_opts(&self, url: impl Into) { + pub fn connect_grpc_opts(&self, url: re_grpc_client::MessageProxyUrl) { if forced_sink_path().is_some() { re_log::debug!("Ignored setting new GrpcSink since {ENV_FORCE_SAVE} is set"); return; @@ -1698,13 +1703,17 @@ impl RecordingStream { return Ok(()); } if forced_sink_path().is_some() { - re_log::debug!("Ignored setting new TcpSink since {ENV_FORCE_SAVE} is set"); + re_log::debug!("Ignored setting new GrpcSink since {ENV_FORCE_SAVE} is set"); return Ok(()); } crate::spawn(opts)?; - self.connect_grpc_opts(format!("http://{}", opts.connect_addr())); + self.connect_grpc_opts( + format!("http://{}", opts.connect_addr()) + .parse() + .expect("should always be valid"), + ); Ok(()) } diff --git a/crates/top/rerun/src/commands/entrypoint.rs b/crates/top/rerun/src/commands/entrypoint.rs index 7280f4a10110..96a1aba14a78 100644 --- a/crates/top/rerun/src/commands/entrypoint.rs +++ b/crates/top/rerun/src/commands/entrypoint.rs @@ -808,7 +808,9 @@ fn run_impl( Ok(()) } else if is_another_viewer_running { // Another viewer is already running on the specified address - let url = format!("http://{server_addr}"); + let url = format!("http://{server_addr}") + .parse() + .expect("should always be valid"); re_log::info!(%url, "Another viewer is already running, streaming data to it."); let sink = re_sdk::sink::GrpcSink::new(url); diff --git a/crates/top/rerun_c/src/lib.rs b/crates/top/rerun_c/src/lib.rs index ea77168a6a4f..971b5095a224 100644 --- a/crates/top/rerun_c/src/lib.rs +++ b/crates/top/rerun_c/src/lib.rs @@ -281,7 +281,7 @@ pub enum CErrorCode { InvalidStringArgument, InvalidEnumValue, InvalidRecordingStreamHandle, - InvalidSocketAddress, + InvalidServerUrl, InvalidComponentTypeHandle, _CategoryRecordingStream = 0x0000_00100, @@ -564,9 +564,12 @@ fn rr_recording_stream_connect_grpc_impl( ) -> Result<(), CError> { let stream = recording_stream(stream)?; - let url = url.as_str("url")?; + let url = url.as_str("url")?.parse(); - stream.connect_grpc_opts(url); + match url { + Ok(url) => stream.connect_grpc_opts(url), + Err(err) => return Err(CError::new(CErrorCode::InvalidServerUrl, &err.to_string())), + } Ok(()) } diff --git a/rerun_cpp/src/rerun/error.hpp b/rerun_cpp/src/rerun/error.hpp index 5787e071e14c..4330d2ae2590 100644 --- a/rerun_cpp/src/rerun/error.hpp +++ b/rerun_cpp/src/rerun/error.hpp @@ -38,7 +38,7 @@ namespace rerun { InvalidStringArgument, InvalidEnumValue, InvalidRecordingStreamHandle, - InvalidSocketAddress, + InvalidServerUrl, InvalidComponentTypeHandle, InvalidTensorDimension, InvalidArchetypeField, diff --git a/rerun_py/Cargo.toml b/rerun_py/Cargo.toml index 60717fd03466..8c24c5c2845a 100644 --- a/rerun_py/Cargo.toml +++ b/rerun_py/Cargo.toml @@ -38,11 +38,11 @@ nasm = ["re_video/nasm"] remote = [ "dep:object_store", "dep:re_protos", - "dep:re_grpc_client", "dep:tokio", "dep:tokio-stream", "dep:tonic", "dep:url", + "re_grpc_client/redap", "re_sdk/grpc", ] @@ -63,7 +63,7 @@ re_build_info.workspace = true re_chunk.workspace = true re_chunk_store.workspace = true re_dataframe.workspace = true -re_grpc_client = { workspace = true, optional = true } +re_grpc_client.workspace = true re_grpc_server = { workspace = true, optional = true } re_log = { workspace = true, features = ["setup"] } re_log_encoding = { workspace = true } diff --git a/rerun_py/src/python_bridge.rs b/rerun_py/src/python_bridge.rs index d1deaf44f8ff..e3844f2c39fa 100644 --- a/rerun_py/src/python_bridge.rs +++ b/rerun_py/src/python_bridge.rs @@ -680,17 +680,20 @@ fn connect_grpc( default_blueprint: Option<&PyMemorySinkStorage>, recording: Option<&PyRecordingStream>, py: Python<'_>, -) { +) -> PyResult<()> { let Some(recording) = get_data_recording(recording) else { - return; + return Ok(()); }; use re_sdk::external::re_grpc_server::DEFAULT_SERVER_PORT; - let url = url.unwrap_or_else(|| format!("http://127.0.0.1:{DEFAULT_SERVER_PORT}")); + let url = url + .unwrap_or_else(|| format!("http://127.0.0.1:{DEFAULT_SERVER_PORT}")) + .parse::() + .map_err(|err| PyRuntimeError::new_err(err.to_string()))?; if re_sdk::forced_sink_path().is_some() { re_log::debug!("Ignored call to `connect()` since _RERUN_TEST_FORCE_SAVE is set"); - return; + return Ok(()); } py.allow_threads(|| { @@ -704,6 +707,8 @@ fn connect_grpc( flush_garbage_queue(); }); + + Ok(()) } #[pyfunction] @@ -717,7 +722,10 @@ fn connect_grpc_blueprint( py: Python<'_>, ) -> PyResult<()> { use re_sdk::external::re_grpc_server::DEFAULT_SERVER_PORT; - let url = url.unwrap_or_else(|| format!("http://127.0.0.1:{DEFAULT_SERVER_PORT}")); + let url = url + .unwrap_or_else(|| format!("http://127.0.0.1:{DEFAULT_SERVER_PORT}")) + .parse::() + .map_err(|err| PyRuntimeError::new_err(err.to_string()))?; if let Some(blueprint_id) = (*blueprint_stream).store_info().map(|info| info.store_id) { // The call to save, needs to flush. From bd1c4aa32c5fdb84d8cf553090ca54e7d014c6a7 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 29 Jan 2025 12:47:23 +0100 Subject: [PATCH 31/87] Remove `connect_tcp` from Python SDK --- rerun_py/src/python_bridge.rs | 89 ----------------------------------- 1 file changed, 89 deletions(-) diff --git a/rerun_py/src/python_bridge.rs b/rerun_py/src/python_bridge.rs index e3844f2c39fa..c0ac940e4936 100644 --- a/rerun_py/src/python_bridge.rs +++ b/rerun_py/src/python_bridge.rs @@ -135,8 +135,6 @@ fn rerun_bindings(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { // sinks m.add_function(wrap_pyfunction!(is_enabled, m)?)?; m.add_function(wrap_pyfunction!(binary_stream, m)?)?; - m.add_function(wrap_pyfunction!(connect_tcp, m)?)?; - m.add_function(wrap_pyfunction!(connect_tcp_blueprint, m)?)?; m.add_function(wrap_pyfunction!(connect_grpc, m)?)?; m.add_function(wrap_pyfunction!(connect_grpc_blueprint, m)?)?; m.add_function(wrap_pyfunction!(save, m)?)?; @@ -586,93 +584,6 @@ fn spawn( re_sdk::spawn(&spawn_opts).map_err(|err| PyRuntimeError::new_err(err.to_string())) } -#[pyfunction] -#[pyo3(signature = (addr = None, flush_timeout_sec=re_sdk::default_flush_timeout().expect("always Some()").as_secs_f32(), default_blueprint = None, recording = None))] -fn connect_tcp( - addr: Option, - flush_timeout_sec: Option, - default_blueprint: Option<&PyMemorySinkStorage>, - recording: Option<&PyRecordingStream>, - py: Python<'_>, -) -> PyResult<()> { - let Some(recording) = get_data_recording(recording) else { - return Ok(()); - }; - - if re_sdk::forced_sink_path().is_some() { - re_log::debug!("Ignored call to `connect()` since _RERUN_TEST_FORCE_SAVE is set"); - return Ok(()); - } - - let addr = if let Some(addr) = addr { - addr.parse()? - } else { - re_sdk::default_server_addr() - }; - - let flush_timeout = flush_timeout_sec.map(std::time::Duration::from_secs_f32); - - // The call to connect may internally flush. - // Release the GIL in case any flushing behavior needs to cleanup a python object. - py.allow_threads(|| { - // We create the sink manually so we can send the default blueprint - // first before the rest of the current recording stream. - let sink = re_sdk::sink::TcpSink::new(addr, flush_timeout); - - if let Some(default_blueprint) = default_blueprint { - send_mem_sink_as_default_blueprint(&sink, default_blueprint); - } - - recording.set_sink(Box::new(sink)); - - flush_garbage_queue(); - }); - - Ok(()) -} - -#[pyfunction] -#[pyo3(signature = (addr, make_active, make_default, blueprint_stream))] -/// Special binding for directly sending a blueprint stream to a connection. -fn connect_tcp_blueprint( - addr: Option, - make_active: bool, - make_default: bool, - blueprint_stream: &PyRecordingStream, - py: Python<'_>, -) -> PyResult<()> { - let addr = if let Some(addr) = addr { - addr.parse()? - } else { - re_sdk::default_server_addr() - }; - - if let Some(blueprint_id) = (*blueprint_stream).store_info().map(|info| info.store_id) { - // The call to save, needs to flush. - // Release the GIL in case any flushing behavior needs to cleanup a python object. - py.allow_threads(|| { - // Flush all the pending blueprint messages before we include the Ready message - blueprint_stream.flush_blocking(); - - let activation_cmd = BlueprintActivationCommand { - blueprint_id, - make_active, - make_default, - }; - - blueprint_stream.record_msg(activation_cmd.into()); - - blueprint_stream.connect_opts(addr, None); - flush_garbage_queue(); - }); - Ok(()) - } else { - Err(PyRuntimeError::new_err( - "Blueprint stream has no store info".to_owned(), - )) - } -} - #[pyfunction] #[pyo3(signature = (url, default_blueprint = None, recording = None))] fn connect_grpc( From 2a126764d85197187099a9b8bc6f2004a07d8c3f Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 29 Jan 2025 13:01:53 +0100 Subject: [PATCH 32/87] Add back `connect` and `spawn` --- crates/top/re_sdk/src/recording_stream.rs | 97 ++++++++++++++++- crates/top/rerun_c/src/lib.rs | 24 +++++ examples/c/minimal/main.c | 4 +- rerun_py/docs/gen_common_index.py | 4 +- rerun_py/rerun_sdk/rerun/__init__.py | 1 - rerun_py/rerun_sdk/rerun/blueprint/api.py | 21 +--- rerun_py/rerun_sdk/rerun/sinks.py | 122 +++------------------- 7 files changed, 139 insertions(+), 134 deletions(-) diff --git a/crates/top/re_sdk/src/recording_stream.rs b/crates/top/re_sdk/src/recording_stream.rs index 8cff390f4eaf..7270b47b398c 100644 --- a/crates/top/re_sdk/src/recording_stream.rs +++ b/crates/top/re_sdk/src/recording_stream.rs @@ -301,6 +301,39 @@ impl RecordingStreamBuilder { Ok((rec, storage)) } + /// Creates a new [`RecordingStream`] that is pre-configured to stream the data through to a + /// remote Rerun instance. + /// + /// See also [`Self::connect_opts`] if you wish to configure the connection. + /// + /// This is an alias for [`Self::connect_grpc`]. + /// + /// ## Example + /// + /// ```no_run + /// let rec = re_sdk::RecordingStreamBuilder::new("rerun_example_app").connect()?; + /// # Ok::<(), Box>(()) + /// ``` + pub fn connect(self) -> RecordingStreamResult { + self.connect_grpc() + } + + /// Creates a new [`RecordingStream`] that is pre-configured to stream the data through to a + /// remote Rerun instance. + /// + /// This is an alias for [`Self::connect_grpc_opts`]. + /// + /// ## Example + /// + /// ```no_run + /// let rec = re_sdk::RecordingStreamBuilder::new("rerun_example_app") + /// .connect_opts("http://127.0.0.1:1852")?; + /// # Ok::<(), Box>(()) + /// ``` + pub fn connect_opts(self, url: impl Into) -> RecordingStreamResult { + self.connect_grpc_opts(url) + } + /// Creates a new [`RecordingStream`] that is pre-configured to stream the data through to a /// remote Rerun instance. /// @@ -416,13 +449,13 @@ impl RecordingStreamBuilder { /// If a Rerun Viewer is already listening on this port, the stream will be redirected to /// that viewer instead of starting a new one. /// - /// See also [`Self::spawn_grpc_opts`] if you wish to configure the behavior of thew Rerun process + /// See also [`Self::spawn_opts`] if you wish to configure the behavior of thew Rerun process /// as well as the underlying connection. /// /// ## Example /// /// ```no_run - /// let rec = re_sdk::RecordingStreamBuilder::new("rerun_example_app").spawn_grpc()?; + /// let rec = re_sdk::RecordingStreamBuilder::new("rerun_example_app").spawn()?; /// # Ok::<(), Box>(()) /// ``` pub fn spawn(self) -> RecordingStreamResult { @@ -436,13 +469,13 @@ impl RecordingStreamBuilder { /// that viewer instead of starting a new one. /// /// The behavior of the spawned Viewer can be configured via `opts`. - /// If you're fine with the default behavior, refer to the simpler [`Self::spawn_grpc`]. + /// If you're fine with the default behavior, refer to the simpler [`Self::spawn`]. /// /// ## Example /// /// ```no_run /// let rec = re_sdk::RecordingStreamBuilder::new("rerun_example_app") - /// .spawn_grpc_opts(&re_sdk::SpawnOptions::default())?; + /// .spawn_opts(&re_sdk::SpawnOptions::default())?; /// # Ok::<(), Box>(()) /// ``` pub fn spawn_opts(self, opts: &crate::SpawnOptions) -> RecordingStreamResult { @@ -1634,6 +1667,28 @@ impl RecordingStream { } impl RecordingStream { + /// Swaps the underlying sink for a [`crate::log_sink::GrpcSink`] sink pre-configured to use + /// the specified address. + /// + /// See also [`Self::connect_opts`] if you wish to configure the connection. + /// + /// This is a convenience wrapper for [`Self::set_sink`] that upholds the same guarantees in + /// terms of data durability and ordering. + /// See [`Self::set_sink`] for more information. + pub fn connect(&self) { + self.connect_grpc(); + } + + /// Swaps the underlying sink for a [`crate::log_sink::GrpcSink`] sink pre-configured to use + /// the specified address. + /// + /// This is a convenience wrapper for [`Self::set_sink`] that upholds the same guarantees in + /// terms of data durability and ordering. + /// See [`Self::set_sink`] for more information. + pub fn connect_opts(&self, url: re_grpc_client::MessageProxyUrl) { + self.connect_grpc_opts(url); + } + /// Swaps the underlying sink for a [`crate::log_sink::GrpcSink`] sink pre-configured to use /// the specified address. /// @@ -1667,6 +1722,40 @@ impl RecordingStream { self.set_sink(Box::new(sink)); } + /// Spawns a new Rerun Viewer process from an executable available in PATH, then swaps the + /// underlying sink for a [`crate::log_sink::GrpcSink`] sink pre-configured to send data to that + /// new process. + /// + /// If a Rerun Viewer is already listening on this port, the stream will be redirected to + /// that viewer instead of starting a new one. + /// + /// See also [`Self::spawn_grpc_opts`] if you wish to configure the behavior of thew Rerun process + /// as well as the underlying connection. + /// + /// This is a convenience wrapper for [`Self::set_sink`] that upholds the same guarantees in + /// terms of data durability and ordering. + /// See [`Self::set_sink`] for more information. + pub fn spawn(&self) -> RecordingStreamResult<()> { + self.spawn_grpc() + } + + /// Spawns a new Rerun Viewer process from an executable available in PATH, then swaps the + /// underlying sink for a [`crate::log_sink::GrpcSink`] sink pre-configured to send data to that + /// new process. + /// + /// If a Rerun Viewer is already listening on this port, the stream will be redirected to + /// that viewer instead of starting a new one. + /// + /// The behavior of the spawned Viewer can be configured via `opts`. + /// If you're fine with the default behavior, refer to the simpler [`Self::spawn`]. + /// + /// This is a convenience wrapper for [`Self::set_sink`] that upholds the same guarantees in + /// terms of data durability and ordering. + /// See [`Self::set_sink`] for more information. + pub fn spawn_opts(&self, opts: &crate::SpawnOptions) -> RecordingStreamResult<()> { + self.spawn_grpc_opts(opts) + } + /// Spawns a new Rerun Viewer process from an executable available in PATH, then swaps the /// underlying sink for a [`crate::log_sink::GrpcSink`] sink pre-configured to send data to that /// new process. diff --git a/crates/top/rerun_c/src/lib.rs b/crates/top/rerun_c/src/lib.rs index 971b5095a224..3e3a1680d17d 100644 --- a/crates/top/rerun_c/src/lib.rs +++ b/crates/top/rerun_c/src/lib.rs @@ -574,6 +574,18 @@ fn rr_recording_stream_connect_grpc_impl( Ok(()) } +#[allow(unsafe_code)] +#[no_mangle] +pub extern "C" fn rr_recording_stream_connect( + id: CRecordingStream, + url: CStringView, + error: *mut CError, +) { + if let Err(err) = rr_recording_stream_connect_grpc_impl(id, url) { + err.write_error(error); + } +} + #[allow(unsafe_code)] #[no_mangle] pub extern "C" fn rr_recording_stream_connect_grpc( @@ -607,6 +619,18 @@ fn rr_recording_stream_spawn_grpc_impl( Ok(()) } +#[allow(unsafe_code)] +#[no_mangle] +pub extern "C" fn rr_recording_stream_spawn( + id: CRecordingStream, + spawn_opts: *const CSpawnOptions, + error: *mut CError, +) { + if let Err(err) = rr_recording_stream_spawn_grpc_impl(id, spawn_opts) { + err.write_error(error); + } +} + #[allow(unsafe_code)] #[no_mangle] pub extern "C" fn rr_recording_stream_spawn_grpc( diff --git a/examples/c/minimal/main.c b/examples/c/minimal/main.c index 7b8b2d2cb0d0..2a445bf4a643 100644 --- a/examples/c/minimal/main.c +++ b/examples/c/minimal/main.c @@ -13,10 +13,10 @@ int main(void) { rr_recording_stream rec = rr_recording_stream_new(&store_info, true, &error); // Connect to running viewer: - //rr_recording_stream_connect(rec, rr_make_string("127.0.0.1:9876"), 2.0f, &error); + //rr_recording_stream_connect(rec, rr_make_string("127.0.0.1:9876"), &error); // Spawn and connect: - rr_recording_stream_spawn(rec, NULL, 2.0f, &error); + rr_recording_stream_spawn(rec, NULL, &error); if (error.code != 0) { printf("Error occurred: %s\n", error.description); diff --git a/rerun_py/docs/gen_common_index.py b/rerun_py/docs/gen_common_index.py index f55c1719449d..37409222cde3 100755 --- a/rerun_py/docs/gen_common_index.py +++ b/rerun_py/docs/gen_common_index.py @@ -10,7 +10,7 @@ Function | Description -------- | ----------- [rerun.init()](initialization/#rerun.init) | Initialize the Rerun SDK … -[rerun.connect_tcp()](initialization/#rerun.connect_tcp) | Connect to a remote Rerun Viewer on the … +[rerun.connect_grpc()](initialization/#rerun.connect_grpc) | Connect to a remote Rerun Viewer on the … [rerun.spawn()](initialization/#rerun.spawn) | Spawn a Rerun Viewer … … @@ -85,7 +85,7 @@ class Section: func_list=[ "init", "connect", - "connect_tcp", + "connect_grpc", "disconnect", "save", "send_blueprint", diff --git a/rerun_py/rerun_sdk/rerun/__init__.py b/rerun_py/rerun_sdk/rerun/__init__.py index 1d01c7102459..8fc545115127 100644 --- a/rerun_py/rerun_sdk/rerun/__init__.py +++ b/rerun_py/rerun_sdk/rerun/__init__.py @@ -174,7 +174,6 @@ from .sinks import ( connect as connect, connect_grpc as connect_grpc, - connect_tcp as connect_tcp, disconnect as disconnect, save as save, send_blueprint as send_blueprint, diff --git a/rerun_py/rerun_sdk/rerun/blueprint/api.py b/rerun_py/rerun_sdk/rerun/blueprint/api.py index 1326c250c8dc..263f59624d20 100644 --- a/rerun_py/rerun_sdk/rerun/blueprint/api.py +++ b/rerun_py/rerun_sdk/rerun/blueprint/api.py @@ -563,20 +563,20 @@ def connect( self, application_id: str, *, - addr: str | None = None, + url: str | None = None, make_active: bool = True, make_default: bool = True, ) -> None: """ - Connect to a remote Rerun Viewer on the given ip:port and send this blueprint. + Connect to a remote Rerun Viewer on the given HTTP(S) URL and send this blueprint. Parameters ---------- application_id: The application ID to use for this blueprint. This must match the application ID used when initiating rerun for any data logging you wish to associate with this blueprint. - addr: - The ip:port to connect to + url: + The HTTP(S) URL to connect to make_active: Immediately make this the active blueprint for the associated `app_id`. Note that setting this to `false` does not mean the blueprint may not still end @@ -589,18 +589,7 @@ def connect( blueprint is currently active. """ - blueprint_stream = RecordingStream( - bindings.new_blueprint( - application_id=application_id, - make_default=False, - make_thread_default=False, - default_enabled=True, - ) - ) - blueprint_stream.set_time_sequence("blueprint", 0) # type: ignore[attr-defined] - self._log_to_stream(blueprint_stream) - - bindings.connect_tcp_blueprint(addr, make_active, make_default, blueprint_stream.to_native()) + return self.connect_grpc(application_id, url=url, make_active=make_active, make_default=make_default) def connect_grpc( self, diff --git a/rerun_py/rerun_sdk/rerun/sinks.py b/rerun_py/rerun_sdk/rerun/sinks.py index 557c49fe34a7..bda63cd7f1b3 100644 --- a/rerun_py/rerun_sdk/rerun/sinks.py +++ b/rerun_py/rerun_sdk/rerun/sinks.py @@ -21,83 +21,21 @@ def is_recording_enabled(recording: RecordingStream | None) -> bool: return bindings.is_enabled() # type: ignore[no-any-return] -@deprecated( - """Please migrate to `rr.connect_tcp(…)`. - See: https://www.rerun.io/docs/reference/migration/migration-0-20 for more details.""" -) def connect( - addr: str | None = None, - *, - flush_timeout_sec: float | None = 2.0, - default_blueprint: BlueprintLike | None = None, - recording: RecordingStream | None = None, -) -> None: - """ - Connect to a remote Rerun Viewer on the given ip:port. - - !!! Warning "Deprecated" - Please migrate to [rerun.connect_tcp][]. - See [the migration guide](https://www.rerun.io/docs/reference/migration/migration-0-20) for more details. - - Requires that you first start a Rerun Viewer by typing 'rerun' in a terminal. - - This function returns immediately. - - Parameters - ---------- - addr: - The ip:port to connect to - flush_timeout_sec: - The minimum time the SDK will wait during a flush before potentially - dropping data if progress is not being made. Passing `None` indicates no timeout, - and can cause a call to `flush` to block indefinitely. - default_blueprint - Optionally set a default blueprint to use for this application. If the application - already has an active blueprint, the new blueprint won't become active until the user - clicks the "reset blueprint" button. If you want to activate the new blueprint - immediately, instead use the [`rerun.send_blueprint`][] API. - recording: - Specifies the [`rerun.RecordingStream`][] to use. - If left unspecified, defaults to the current active data recording, if there is one. - See also: [`rerun.init`][], [`rerun.set_global_data_recording`][]. - - """ - - warnings.warn( - message=("`connect` is deprecated. Use `connect_tcp` instead."), - category=DeprecationWarning, - ) - - return connect_tcp( - addr, - flush_timeout_sec=flush_timeout_sec, - default_blueprint=default_blueprint, - recording=recording, # NOLINT - ) - - -def connect_tcp( - addr: str | None = None, + url: str | None = None, *, - flush_timeout_sec: float | None = 2.0, default_blueprint: BlueprintLike | None = None, recording: RecordingStream | None = None, ) -> None: """ - Connect to a remote Rerun Viewer on the given ip:port. - - Requires that you first start a Rerun Viewer by typing 'rerun' in a terminal. + Connect to a remote Rerun Viewer on the given HTTP(S) URL. This function returns immediately. Parameters ---------- - addr: - The ip:port to connect to - flush_timeout_sec: - The minimum time the SDK will wait during a flush before potentially - dropping data if progress is not being made. Passing `None` indicates no timeout, - and can cause a call to `flush` to block indefinitely. + url: + The HTTP(S) URL to connect to default_blueprint Optionally set a default blueprint to use for this application. If the application already has an active blueprint, the new blueprint won't become active until the user @@ -109,33 +47,7 @@ def connect_tcp( See also: [`rerun.init`][], [`rerun.set_global_data_recording`][]. """ - - if not is_recording_enabled(recording): - logging.warning("Rerun is disabled - connect() call ignored") - return - - application_id = get_application_id(recording=recording) # NOLINT - if application_id is None: - raise ValueError( - "No application id found. You must call rerun.init before connecting to a viewer, or provide a recording." - ) - - # If a blueprint is provided, we need to create a blueprint storage object - blueprint_storage = None - if default_blueprint is not None: - blueprint_storage = create_in_memory_blueprint( - application_id=application_id, blueprint=default_blueprint - ).storage - - bindings.connect_tcp( - addr=addr, - flush_timeout_sec=flush_timeout_sec, - default_blueprint=blueprint_storage, - recording=recording.to_native() if recording is not None else None, - ) - - -_is_connect_grpc_available = hasattr(bindings, "connect_grpc") + return connect_grpc(url, default_blueprint=default_blueprint, recording=recording) def connect_grpc( @@ -164,10 +76,6 @@ def connect_grpc( See also: [`rerun.init`][], [`rerun.set_global_data_recording`][]. """ - - if not _is_connect_grpc_available: - raise NotImplementedError("`rerun_sdk` was compiled without `remote` feature, connect_grpc is not available") - if not is_recording_enabled(recording): logging.warning("Rerun is disabled - connect() call ignored") return @@ -538,18 +446,14 @@ def spawn( """ - if not is_recording_enabled(recording): - logging.warning("Rerun is disabled - spawn() call ignored.") - return - - _spawn_viewer(port=port, memory_limit=memory_limit, hide_welcome_screen=hide_welcome_screen) - - if connect: - connect_tcp( - f"127.0.0.1:{port}", - recording=recording, # NOLINT - default_blueprint=default_blueprint, - ) + return spawn_grpc( + port=port, + connect=connect, + memory_limit=memory_limit, + hide_welcome_screen=hide_welcome_screen, + default_blueprint=default_blueprint, + recording=recording, + ) def spawn_grpc( From 4ff359be8fc8493b18cd16403eade499c52296b0 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 29 Jan 2025 13:01:58 +0100 Subject: [PATCH 33/87] Revert quick start spawn example --- .../re_viewer/data/quick_start_guides/quick_start_spawn.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/viewer/re_viewer/data/quick_start_guides/quick_start_spawn.py b/crates/viewer/re_viewer/data/quick_start_guides/quick_start_spawn.py index f9bf11fcf17a..d108fc4d6d08 100644 --- a/crates/viewer/re_viewer/data/quick_start_guides/quick_start_spawn.py +++ b/crates/viewer/re_viewer/data/quick_start_guides/quick_start_spawn.py @@ -4,8 +4,7 @@ import rerun as rr # Initialize the SDK, give our recording a unique name, and spawn a viewer -rr.init("rerun_example_quick_start_spawn") -rr.serve_web(open_browser=True) +rr.init("rerun_example_quick_start_spawn", spawn=True) # Create some data SIZE = 10 From b25a08dc7894ddd6a6ee572c5d1e79a5c3da92ab Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 29 Jan 2025 13:02:37 +0100 Subject: [PATCH 34/87] Update custom callback example --- examples/rust/custom_callback/Cargo.toml | 1 + examples/rust/custom_callback/src/app.rs | 10 ++-------- examples/rust/custom_callback/src/viewer.rs | 16 +++++++--------- 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/examples/rust/custom_callback/Cargo.toml b/examples/rust/custom_callback/Cargo.toml index 25300968d4a8..56d7e9b3e8e6 100644 --- a/examples/rust/custom_callback/Cargo.toml +++ b/examples/rust/custom_callback/Cargo.toml @@ -27,6 +27,7 @@ rerun = { path = "../../../crates/top/rerun", features = [ "native_viewer", "run", ] } +re_grpc_server = { path = "../../../crates/store/re_grpc_server" } bincode = "1.3.3" mimalloc = "0.1.43" diff --git a/examples/rust/custom_callback/src/app.rs b/examples/rust/custom_callback/src/app.rs index cff9402f5b8b..aaee7e3991ea 100644 --- a/examples/rust/custom_callback/src/app.rs +++ b/examples/rust/custom_callback/src/app.rs @@ -1,10 +1,7 @@ //! The external application that will be controlled by the extended viewer ui. use core::f32; -use std::{ - f32::consts::{PI, TAU}, - net::ToSocketAddrs, -}; +use std::f32::consts::{PI, TAU}; use custom_callback::comms::{app::ControlApp, protocol::Message}; @@ -18,10 +15,7 @@ use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; async fn main() -> Result<(), Box> { let mut app = ControlApp::bind("127.0.0.1:8888").await?.run(); let rec = rerun::RecordingStreamBuilder::new("rerun_example_custom_callback") - .connect_tcp_opts( - "127.0.0.1:9877".to_socket_addrs().unwrap().next().unwrap(), - None, - )?; + .connect_opts("http://127.0.0.1:1853")?; // Add a handler for incoming messages let add_rec = rec.clone(); diff --git a/examples/rust/custom_callback/src/viewer.rs b/examples/rust/custom_callback/src/viewer.rs index dc9110acf250..06fe61fd24b9 100644 --- a/examples/rust/custom_callback/src/viewer.rs +++ b/examples/rust/custom_callback/src/viewer.rs @@ -1,8 +1,6 @@ use custom_callback::{comms::viewer::ControlViewer, panel::Control}; -use rerun::external::{eframe, re_log, re_memory, re_sdk_comms, re_viewer}; - -use std::net::Ipv4Addr; +use rerun::external::{eframe, re_log, re_memory, re_viewer}; // By using `re_memory::AccountingAllocator` Rerun can keep track of exactly how much memory it is using, // and prune the data store when it goes above a certain limit. @@ -24,13 +22,13 @@ async fn main() -> Result<(), Box> { // them to Rerun analytics (if the `analytics` feature is on in `Cargo.toml`). re_crash_handler::install_crash_handlers(re_viewer::build_info()); - // Listen for TCP connections from Rerun's logging SDKs. + // Listen for gRPC connections from Rerun's logging SDKs. // There are other ways of "feeding" the viewer though - all you need is a `re_smart_channel::Receiver`. - let rx = re_sdk_comms::serve( - &Ipv4Addr::UNSPECIFIED.to_string(), - re_sdk_comms::DEFAULT_SERVER_PORT + 1, - Default::default(), - )?; + let rx = re_grpc_server::spawn_with_recv( + "0.0.0.0:1853".parse()?, + "75%".parse()?, + re_grpc_server::shutdown::never(), + ); // First we attempt to connect to the external application let viewer = ControlViewer::connect(format!("127.0.0.1:{CONTROL_PORT}")).await?; From 12b3199b2be76cc48a34b4635e7c6ca6f7b79629 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 29 Jan 2025 13:02:45 +0100 Subject: [PATCH 35/87] Update extend viewer example --- examples/rust/extend_viewer_ui/Cargo.toml | 17 +++++++++++++---- examples/rust/extend_viewer_ui/src/main.rs | 15 ++++++++------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/examples/rust/extend_viewer_ui/Cargo.toml b/examples/rust/extend_viewer_ui/Cargo.toml index 633cdb891f63..bdef41504847 100644 --- a/examples/rust/extend_viewer_ui/Cargo.toml +++ b/examples/rust/extend_viewer_ui/Cargo.toml @@ -16,10 +16,19 @@ analytics = ["re_crash_handler/analytics", "re_viewer/analytics"] re_crash_handler = { path = "../../../crates/utils/re_crash_handler" } re_viewer = { path = "../../../crates/viewer/re_viewer", default-features = false } -# We need re_sdk_comms to receive log events from and SDK: -re_sdk_comms = { path = "../../../crates/store/re_sdk_comms", features = [ - "server", -] } +# We need re_grpc_server to receive log events from an SDK: +re_grpc_server = { path = "../../../crates/store/re_grpc_server" } # mimalloc is a much faster allocator: mimalloc = "0.1.43" + +# We need `tokio` to be able to run `re_grpc_server`: +tokio = { version = "1.14.0", features = [ + "macros", + "rt-multi-thread", + "time", + "net", + "io-util", + "sync", + "signal", +] } diff --git a/examples/rust/extend_viewer_ui/src/main.rs b/examples/rust/extend_viewer_ui/src/main.rs index 8e5b5685ca4e..8b65784a856f 100644 --- a/examples/rust/extend_viewer_ui/src/main.rs +++ b/examples/rust/extend_viewer_ui/src/main.rs @@ -11,7 +11,8 @@ use re_viewer::external::{ static GLOBAL: re_memory::AccountingAllocator = re_memory::AccountingAllocator::new(mimalloc::MiMalloc); -fn main() -> Result<(), Box> { +#[tokio::main] +async fn main() -> Result<(), Box> { let main_thread_token = re_viewer::MainThreadToken::i_promise_i_am_on_the_main_thread(); // Direct calls using the `log` crate to stderr. Control with `RUST_LOG=debug` etc. @@ -21,13 +22,13 @@ fn main() -> Result<(), Box> { // them to Rerun analytics (if the `analytics` feature is on in `Cargo.toml`). re_crash_handler::install_crash_handlers(re_viewer::build_info()); - // Listen for TCP connections from Rerun's logging SDKs. + // Listen for gRPC connections from Rerun's logging SDKs. // There are other ways of "feeding" the viewer though - all you need is a `re_smart_channel::Receiver`. - let rx = re_sdk_comms::serve( - "0.0.0.0", - re_sdk_comms::DEFAULT_SERVER_PORT, - Default::default(), - )?; + let rx = re_grpc_server::spawn_with_recv( + "0.0.0.0:1852".parse()?, + "75%".parse()?, + re_grpc_server::shutdown::never(), + ); let mut native_options = re_viewer::native::eframe_options(None); native_options.viewport = native_options From 730853a5de6be1f73ee03a07bb2afb5de58c9f4c Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 29 Jan 2025 13:02:52 +0100 Subject: [PATCH 36/87] Update custom view example --- examples/rust/custom_view/Cargo.toml | 17 +++++++++++++---- examples/rust/custom_view/src/main.rs | 15 ++++++++------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/examples/rust/custom_view/Cargo.toml b/examples/rust/custom_view/Cargo.toml index cc38521c6e30..8c325abe6f6a 100644 --- a/examples/rust/custom_view/Cargo.toml +++ b/examples/rust/custom_view/Cargo.toml @@ -16,10 +16,19 @@ analytics = ["re_crash_handler/analytics", "re_viewer/analytics"] re_crash_handler = { path = "../../../crates/utils/re_crash_handler" } re_viewer = { path = "../../../crates/viewer/re_viewer", default-features = false } -# We need re_sdk_comms to receive log events from and SDK: -re_sdk_comms = { path = "../../../crates/store/re_sdk_comms", features = [ - "server", -] } +# We need re_grpc_server to receive log events from an SDK: +re_grpc_server = { path = "../../../crates/store/re_grpc_server" } # mimalloc is a much faster allocator: mimalloc = "0.1.43" + +# We need `tokio` to be able to run `re_grpc_server`: +tokio = { version = "1.14.0", features = [ + "macros", + "rt-multi-thread", + "time", + "net", + "io-util", + "sync", + "signal", +] } diff --git a/examples/rust/custom_view/src/main.rs b/examples/rust/custom_view/src/main.rs index cc7745bf7930..0fd9d385fd7a 100644 --- a/examples/rust/custom_view/src/main.rs +++ b/examples/rust/custom_view/src/main.rs @@ -12,7 +12,8 @@ mod color_coordinates_visualizer_system; static GLOBAL: re_memory::AccountingAllocator = re_memory::AccountingAllocator::new(mimalloc::MiMalloc); -fn main() -> Result<(), Box> { +#[tokio::main] +async fn main() -> Result<(), Box> { let main_thread_token = re_viewer::MainThreadToken::i_promise_i_am_on_the_main_thread(); // Direct calls using the `log` crate to stderr. Control with `RUST_LOG=debug` etc. @@ -22,13 +23,13 @@ fn main() -> Result<(), Box> { // them to Rerun analytics (if the `analytics` feature is on in `Cargo.toml`). re_crash_handler::install_crash_handlers(re_viewer::build_info()); - // Listen for TCP connections from Rerun's logging SDKs. + // Listen for gRPC connections from Rerun's logging SDKs. // There are other ways of "feeding" the viewer though - all you need is a `re_smart_channel::Receiver`. - let rx = re_sdk_comms::serve( - "0.0.0.0", - re_sdk_comms::DEFAULT_SERVER_PORT, - Default::default(), - )?; + let rx = re_grpc_server::spawn_with_recv( + "0.0.0.0:1852".parse()?, + "75%".parse()?, + re_grpc_server::shutdown::never(), + ); let startup_options = re_viewer::StartupOptions::default(); From e60055174af9e05aa0b39bbe49ee2d2197c05ba7 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 29 Jan 2025 13:04:12 +0100 Subject: [PATCH 37/87] Update usages of `connect` and `spawn` --- Cargo.lock | 9 ++--- crates/top/rerun/src/clap.rs | 2 +- .../quick_start_guides/quick_start_connect.rs | 3 +- .../all/concepts/app-model/native-sync.py | 5 ++- .../all/concepts/app-model/native-sync.rs | 2 +- .../all/quick_start/quick_start_connect.py | 4 +-- .../all/quick_start/quick_start_connect.rs | 3 +- docs/snippets/all/tutorials/data_out.py | 2 +- .../multiprocess_logging.py | 2 +- rerun_cpp/docs/readme_snippets.cpp | 4 +-- rerun_cpp/src/rerun/recording_stream.cpp | 14 ++++++++ rerun_cpp/src/rerun/recording_stream.hpp | 36 +++---------------- rerun_cpp/tests/recording_stream.cpp | 2 +- tests/cpp/plot_dashboard_stress/main.cpp | 4 +-- tests/python/gil_stress/main.py | 2 +- .../rust/test_data_density_graph/src/main.rs | 25 ++++++------- 16 files changed, 51 insertions(+), 68 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7246a2419063..37c164df4df9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1740,6 +1740,7 @@ dependencies = [ "parking_lot", "re_crash_handler", "re_error", + "re_grpc_server", "rerun", "serde", "tokio", @@ -1767,8 +1768,9 @@ version = "0.22.0-alpha.1+dev" dependencies = [ "mimalloc", "re_crash_handler", - "re_sdk_comms", + "re_grpc_server", "re_viewer", + "tokio", ] [[package]] @@ -2391,8 +2393,9 @@ version = "0.22.0-alpha.1+dev" dependencies = [ "mimalloc", "re_crash_handler", - "re_sdk_comms", + "re_grpc_server", "re_viewer", + "tokio", ] [[package]] @@ -6438,7 +6441,6 @@ dependencies = [ "re_log_encoding", "re_log_types", "re_memory", - "re_sdk_comms", "re_smart_channel", "re_types_core", "re_web_viewer_server", @@ -7022,7 +7024,6 @@ dependencies = [ "re_memory", "re_query", "re_renderer", - "re_sdk_comms", "re_selection_panel", "re_smart_channel", "re_time_panel", diff --git a/crates/top/rerun/src/clap.rs b/crates/top/rerun/src/clap.rs index d8a9966df2e6..960263ab243d 100644 --- a/crates/top/rerun/src/clap.rs +++ b/crates/top/rerun/src/clap.rs @@ -111,7 +111,7 @@ impl RerunArgs { )), RerunBehavior::Connect(url) => Ok(( - RecordingStreamBuilder::new(application_id).connect_grpc_opts(url)?, + RecordingStreamBuilder::new(application_id).connect_opts(url)?, Default::default(), )), diff --git a/crates/viewer/re_viewer/data/quick_start_guides/quick_start_connect.rs b/crates/viewer/re_viewer/data/quick_start_guides/quick_start_connect.rs index 988f0ba019c6..0ccd3064e122 100644 --- a/crates/viewer/re_viewer/data/quick_start_guides/quick_start_connect.rs +++ b/crates/viewer/re_viewer/data/quick_start_guides/quick_start_connect.rs @@ -4,8 +4,7 @@ use rerun::{demo_util::grid, external::glam}; fn main() -> Result<(), Box> { // Create a new `RecordingStream` which sends data over TCP to the viewer process. - let rec = - rerun::RecordingStreamBuilder::new("rerun_example_quick_start_connect").connect_tcp()?; + let rec = rerun::RecordingStreamBuilder::new("rerun_example_quick_start_connect").connect()?; // Create some data using the `grid` utility function. let points = grid(glam::Vec3::splat(-10.0), glam::Vec3::splat(10.0), 10); diff --git a/docs/snippets/all/concepts/app-model/native-sync.py b/docs/snippets/all/concepts/app-model/native-sync.py index 831ff39a7dd0..cefb40529346 100755 --- a/docs/snippets/all/concepts/app-model/native-sync.py +++ b/docs/snippets/all/concepts/app-model/native-sync.py @@ -4,9 +4,8 @@ rr.init("rerun_example_native_sync") -# Connect to the Rerun TCP server using the default address and -# port: localhost:9876 -rr.connect_tcp() +# Connect to the Rerun gRPC server using the default address and url: http://localhost:1852 +rr.connect_grpc() # Log data as usual, thereby pushing it into the TCP socket. while True: diff --git a/docs/snippets/all/concepts/app-model/native-sync.rs b/docs/snippets/all/concepts/app-model/native-sync.rs index 9ee90e0ea1ab..2db8313dd9df 100644 --- a/docs/snippets/all/concepts/app-model/native-sync.rs +++ b/docs/snippets/all/concepts/app-model/native-sync.rs @@ -1,7 +1,7 @@ fn main() -> Result<(), Box> { // Connect to the Rerun TCP server using the default address and // port: localhost:9876 - let rec = rerun::RecordingStreamBuilder::new("rerun_example_native_sync").connect_tcp()?; + let rec = rerun::RecordingStreamBuilder::new("rerun_example_native_sync").connect()?; // Log data as usual, thereby pushing it into the TCP socket. loop { diff --git a/docs/snippets/all/quick_start/quick_start_connect.py b/docs/snippets/all/quick_start/quick_start_connect.py index 204393418803..5ac624d72cc9 100644 --- a/docs/snippets/all/quick_start/quick_start_connect.py +++ b/docs/snippets/all/quick_start/quick_start_connect.py @@ -6,8 +6,8 @@ # Initialize the SDK and give our recording a unique name rr.init("rerun_example_quick_start_connect") -# Connect to a local viewer using the default port -rr.connect_tcp() +# Connect to a local viewer using the default URL +rr.connect_grpc() # Create some data diff --git a/docs/snippets/all/quick_start/quick_start_connect.rs b/docs/snippets/all/quick_start/quick_start_connect.rs index 988f0ba019c6..0ccd3064e122 100644 --- a/docs/snippets/all/quick_start/quick_start_connect.rs +++ b/docs/snippets/all/quick_start/quick_start_connect.rs @@ -4,8 +4,7 @@ use rerun::{demo_util::grid, external::glam}; fn main() -> Result<(), Box> { // Create a new `RecordingStream` which sends data over TCP to the viewer process. - let rec = - rerun::RecordingStreamBuilder::new("rerun_example_quick_start_connect").connect_tcp()?; + let rec = rerun::RecordingStreamBuilder::new("rerun_example_quick_start_connect").connect()?; // Create some data using the `grid` utility function. let points = grid(glam::Vec3::splat(-10.0), glam::Vec3::splat(10.0), 10); diff --git a/docs/snippets/all/tutorials/data_out.py b/docs/snippets/all/tutorials/data_out.py index b0a81fabd0ce..659a83a4f023 100644 --- a/docs/snippets/all/tutorials/data_out.py +++ b/docs/snippets/all/tutorials/data_out.py @@ -27,7 +27,7 @@ # Connect to the viewer rr.init(recording.application_id(), recording_id=recording.recording_id()) -rr.connect_tcp() +rr.connect_grpc() # log the jaw open state signal as a scalar rr.send_columns( diff --git a/examples/python/multiprocess_logging/multiprocess_logging.py b/examples/python/multiprocess_logging/multiprocess_logging.py index 7721ff4794ab..b1c4bf4c04da 100755 --- a/examples/python/multiprocess_logging/multiprocess_logging.py +++ b/examples/python/multiprocess_logging/multiprocess_logging.py @@ -24,7 +24,7 @@ def task(child_index: int) -> None: rr.init("rerun_example_multiprocessing") # We then have to connect to the viewer instance. - rr.connect_tcp() + rr.connect_grpc() title = f"task_{child_index}" rr.log( diff --git a/rerun_cpp/docs/readme_snippets.cpp b/rerun_cpp/docs/readme_snippets.cpp index 5d79c9276416..b4e676dd4431 100644 --- a/rerun_cpp/docs/readme_snippets.cpp +++ b/rerun_cpp/docs/readme_snippets.cpp @@ -21,7 +21,7 @@ static std::vector create_image() { // Create a recording stream. rerun::RecordingStream rec("rerun_example_app"); // Spawn the viewer and connect to it. - rec.spawn().exit_on_failure(); + rec.spawn_grpc().exit_on_failure(); std::vector points = create_positions(); std::vector colors = create_colors(); @@ -63,7 +63,7 @@ static std::vector create_image() { rec.log("path/to/points", rerun::Points3D(points).with_colors(colors)); // Spawn & connect later. - auto result = rec.spawn(); + auto result = rec.spawn_grpc(); if (result.is_err()) { // Handle error. } diff --git a/rerun_cpp/src/rerun/recording_stream.cpp b/rerun_cpp/src/rerun/recording_stream.cpp index 4feac124deda..08355863928b 100644 --- a/rerun_cpp/src/rerun/recording_stream.cpp +++ b/rerun_cpp/src/rerun/recording_stream.cpp @@ -104,6 +104,20 @@ namespace rerun { } } + Error RecordingStream::connect(std::string_view url) const { + return connect_grpc(url); + } + + Error RecordingStream::connect_grpc(std::string_view url) const { + rr_error status = {}; + rr_recording_stream_connect_grpc(_id, detail::to_rr_string(url), &status); + return status; + } + + Error RecordingStream::spawn(const SpawnOptions& options) const { + return spawn_grpc(options); + } + Error RecordingStream::spawn_grpc(const SpawnOptions& options) const { rr_spawn_options rerun_c_options = {}; options.fill_rerun_c_struct(rerun_c_options); diff --git a/rerun_cpp/src/rerun/recording_stream.hpp b/rerun_cpp/src/rerun/recording_stream.hpp index a152a21a73db..7dde4d4e790a 100644 --- a/rerun_cpp/src/rerun/recording_stream.hpp +++ b/rerun_cpp/src/rerun/recording_stream.hpp @@ -133,33 +133,12 @@ namespace rerun { /// \details Either of these needs to be called, otherwise the stream will buffer up indefinitely. /// @{ - /// Connect to a remote Rerun Viewer on the given ip:port. - /// - /// Requires that you first start a Rerun Viewer by typing 'rerun' in a terminal. - /// - /// flush_timeout_sec: - /// The minimum time the SDK will wait during a flush before potentially - /// dropping data if progress is not being made. Passing a negative value indicates no - /// timeout, and can cause a call to `flush` to block indefinitely. - /// - /// This function returns immediately. - [[deprecated("Use `connect_tcp` instead")]] Error connect( - std::string_view tcp_addr = "127.0.0.1:9876", float flush_timeout_sec = 2.0 - ) const; - - /// Connect to a remote Rerun Viewer on the given ip:port. + /// Connect to a remote Rerun Viewer on the given HTTP(S) URL. /// /// Requires that you first start a Rerun Viewer by typing 'rerun' in a terminal. /// - /// flush_timeout_sec: - /// The minimum time the SDK will wait during a flush before potentially - /// dropping data if progress is not being made. Passing a negative value indicates no - /// timeout, and can cause a call to `flush` to block indefinitely. - /// /// This function returns immediately. - Error connect_tcp( - std::string_view tcp_addr = "127.0.0.1:9876", float flush_timeout_sec = 2.0 - ) const; + Error connect(std::string_view url = "http://127.0.0.1:1852") const; /// Connect to a remote Rerun Viewer on the given HTTP(S) URL. /// @@ -169,20 +148,15 @@ namespace rerun { Error connect_grpc(std::string_view url = "http://127.0.0.1:1852") const; /// Spawns a new Rerun Viewer process from an executable available in PATH, then connects to it - /// over TCP. + /// over gRPC. /// - /// If a Rerun Viewer is already listening on this TCP port, the stream will be redirected to + /// If a Rerun Viewer is already listening on this port, the stream will be redirected to /// that viewer instead of starting a new one. /// /// ## Parameters /// options: /// See `rerun::SpawnOptions` for more information. - /// - /// flush_timeout_sec: - /// The minimum time the SDK will wait during a flush before potentially - /// dropping data if progress is not being made. Passing a negative value indicates no - /// timeout, and can cause a call to `flush` to block indefinitely. - Error spawn(const SpawnOptions& options = {}, float flush_timeout_sec = 2.0) const; + Error spawn(const SpawnOptions& options = {}) const; /// Spawns a new Rerun Viewer process from an executable available in PATH, then connects to it /// over gRPC. diff --git a/rerun_cpp/tests/recording_stream.cpp b/rerun_cpp/tests/recording_stream.cpp index 0dda46ca45c7..87dd5b913b8e 100644 --- a/rerun_cpp/tests/recording_stream.cpp +++ b/rerun_cpp/tests/recording_stream.cpp @@ -381,7 +381,7 @@ void test_logging_to_connection(const char* address, const rerun::RecordingStrea THEN("then the save call fails") { CHECK( stream.connect_grpc("definitely not valid!").code == - rerun::ErrorCode::InvalidSocketAddress + rerun::ErrorCode::InvalidServerUrl ); } } diff --git a/tests/cpp/plot_dashboard_stress/main.cpp b/tests/cpp/plot_dashboard_stress/main.cpp index 11c2f3328ed0..e8b26c35b0d4 100644 --- a/tests/cpp/plot_dashboard_stress/main.cpp +++ b/tests/cpp/plot_dashboard_stress/main.cpp @@ -57,7 +57,7 @@ int main(int argc, char** argv) { // TODO(#4602): need common rerun args helper library if (args["spawn"].as()) { - rec.spawn().exit_on_failure(); + rec.spawn_grpc().exit_on_failure(); } else if (args["connect"].as()) { rec.connect_grpc().exit_on_failure(); } else if (args["stdout"].as()) { @@ -65,7 +65,7 @@ int main(int argc, char** argv) { } else if (args.count("save")) { rec.save(args["save"].as()).exit_on_failure(); } else { - rec.spawn().exit_on_failure(); + rec.spawn_grpc().exit_on_failure(); } const auto num_plots = args["num-plots"].as(); diff --git a/tests/python/gil_stress/main.py b/tests/python/gil_stress/main.py index bd1d3c44c52a..085a63ab6422 100644 --- a/tests/python/gil_stress/main.py +++ b/tests/python/gil_stress/main.py @@ -35,7 +35,7 @@ rr.log("test", rr.Points3D([1, 2, 3]), recording=rec) rec = rr.new_recording(application_id="test") -rr.connect_tcp(recording=rec) +rr.connect_grpc(recording=rec) rr.log("test", rr.Points3D([1, 2, 3]), recording=rec) rec = rr.new_recording(application_id="test") diff --git a/tests/rust/test_data_density_graph/src/main.rs b/tests/rust/test_data_density_graph/src/main.rs index e962ff226492..e3a833fccb03 100644 --- a/tests/rust/test_data_density_graph/src/main.rs +++ b/tests/rust/test_data_density_graph/src/main.rs @@ -14,21 +14,18 @@ fn main() -> anyhow::Result<()> { re_log::setup_logging(); let rec = rerun::RecordingStreamBuilder::new("rerun_example_test_data_density_graph") - .spawn_opts( - &rerun::SpawnOptions { - wait_for_bind: true, - extra_env: { - use re_chunk_store::ChunkStoreConfig as C; - vec![ - (C::ENV_CHUNK_MAX_BYTES.into(), "0".into()), - (C::ENV_CHUNK_MAX_ROWS.into(), "0".into()), - (C::ENV_CHUNK_MAX_ROWS_IF_UNSORTED.into(), "0".into()), - ] - }, - ..Default::default() + .spawn_opts(&rerun::SpawnOptions { + wait_for_bind: true, + extra_env: { + use re_chunk_store::ChunkStoreConfig as C; + vec![ + (C::ENV_CHUNK_MAX_BYTES.into(), "0".into()), + (C::ENV_CHUNK_MAX_ROWS.into(), "0".into()), + (C::ENV_CHUNK_MAX_ROWS_IF_UNSORTED.into(), "0".into()), + ] }, - rerun::default_flush_timeout(), - )?; + ..Default::default() + })?; run(&rec) } From 8585f46cc6a7bd514c5d062f9103ea8bc56008e0 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 29 Jan 2025 13:04:54 +0100 Subject: [PATCH 38/87] Delete `re_sdk_comms` --- ARCHITECTURE.md | 1 - Cargo.lock | 16 - Cargo.toml | 1 - crates/store/re_sdk_comms/Cargo.toml | 49 --- crates/store/re_sdk_comms/README.md | 10 - .../store/re_sdk_comms/src/buffered_client.rs | 327 -------------- crates/store/re_sdk_comms/src/lib.rs | 83 ---- crates/store/re_sdk_comms/src/server.rs | 412 ------------------ crates/store/re_sdk_comms/src/tcp_client.rs | 196 --------- 9 files changed, 1095 deletions(-) delete mode 100644 crates/store/re_sdk_comms/Cargo.toml delete mode 100644 crates/store/re_sdk_comms/README.md delete mode 100644 crates/store/re_sdk_comms/src/buffered_client.rs delete mode 100644 crates/store/re_sdk_comms/src/lib.rs delete mode 100644 crates/store/re_sdk_comms/src/server.rs delete mode 100644 crates/store/re_sdk_comms/src/tcp_client.rs diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 6fabb520e56c..0535fe7d262f 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -194,7 +194,6 @@ Update instructions: | re_data_source | Handles loading of Rerun data from different sources | | re_grpc_client | Communicate with the Rerun Data Platform over gRPC | | re_grpc_server | Host an in-memory Storage Node | -| re_sdk_comms | TCP communication between Rerun SDK and Rerun Server | | re_web_viewer_server | Serves the Rerun web viewer (Wasm and HTML) over HTTP | ### Build support diff --git a/Cargo.lock b/Cargo.lock index 37c164df4df9..fc8cf2b89273 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6450,22 +6450,6 @@ dependencies = [ "webbrowser", ] -[[package]] -name = "re_sdk_comms" -version = "0.22.0-alpha.1+dev" -dependencies = [ - "ahash", - "crossbeam", - "document-features", - "rand", - "re_build_info", - "re_log", - "re_log_encoding", - "re_log_types", - "re_smart_channel", - "thiserror 1.0.65", -] - [[package]] name = "re_selection_panel" version = "0.22.0-alpha.1+dev" diff --git a/Cargo.toml b/Cargo.toml index 414194d5eefd..2edefecd5699 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,7 +60,6 @@ re_protos = { path = "crates/store/re_protos", version = "=0.22.0-alpha.1", defa re_log_encoding = { path = "crates/store/re_log_encoding", version = "=0.22.0-alpha.1", default-features = false } re_log_types = { path = "crates/store/re_log_types", version = "=0.22.0-alpha.1", default-features = false } re_query = { path = "crates/store/re_query", version = "=0.22.0-alpha.1", default-features = false } -re_sdk_comms = { path = "crates/store/re_sdk_comms", version = "=0.22.0-alpha.1", default-features = false } re_sorbet = { path = "crates/store/re_sorbet", version = "=0.22.0-alpha.1", default-features = false } re_types = { path = "crates/store/re_types", version = "=0.22.0-alpha.1", default-features = false } re_types_core = { path = "crates/store/re_types_core", version = "=0.22.0-alpha.1", default-features = false } diff --git a/crates/store/re_sdk_comms/Cargo.toml b/crates/store/re_sdk_comms/Cargo.toml deleted file mode 100644 index 85c840f7b4b1..000000000000 --- a/crates/store/re_sdk_comms/Cargo.toml +++ /dev/null @@ -1,49 +0,0 @@ -[package] -name = "re_sdk_comms" -authors.workspace = true -description = "TCP communication between Rerun SDK and Rerun Server" -edition.workspace = true -homepage.workspace = true -include.workspace = true -license.workspace = true -publish = true -readme = "README.md" -repository.workspace = true -rust-version.workspace = true -version.workspace = true - -[lints] -workspace = true - -[package.metadata.docs.rs] -all-features = true - - -[features] -## Enable the client (SDK-side). -client = ["re_log_encoding/encoder"] - -## Enable the server. -server = ["rand", "re_log_encoding/decoder"] - - -[dependencies] -re_build_info.workspace = true -re_log_encoding.workspace = true -re_log_types.workspace = true -re_log.workspace = true -re_smart_channel.workspace = true - -ahash.workspace = true -crossbeam.workspace = true -document-features.workspace = true -thiserror.workspace = true - -# Optional dependencies: - -# We use rand for the congestion manager -rand = { workspace = true, optional = true, features = [ - "std", - "std_rng", - "small_rng", -] } diff --git a/crates/store/re_sdk_comms/README.md b/crates/store/re_sdk_comms/README.md deleted file mode 100644 index 0c2cfbfbe432..000000000000 --- a/crates/store/re_sdk_comms/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# re_sdk_comms - -Part of the [`rerun`](https://github.com/rerun-io/rerun) family of crates. - -[![Latest version](https://img.shields.io/crates/v/re_sdk_comms.svg)](https://crates.io/crates/store/re_sdk_comms) -[![Documentation](https://docs.rs/re_sdk_comms/badge.svg)](https://docs.rs/re_sdk_comms) -![MIT](https://img.shields.io/badge/license-MIT-blue.svg) -![Apache](https://img.shields.io/badge/license-Apache-blue.svg) - -TCP communication between Rerun SDK and Rerun Server. diff --git a/crates/store/re_sdk_comms/src/buffered_client.rs b/crates/store/re_sdk_comms/src/buffered_client.rs deleted file mode 100644 index 4616bcce5f7f..000000000000 --- a/crates/store/re_sdk_comms/src/buffered_client.rs +++ /dev/null @@ -1,327 +0,0 @@ -use std::{fmt, net::SocketAddr, thread::JoinHandle}; - -use crossbeam::channel::{select, Receiver, Sender}; - -use re_log_types::LogMsg; - -#[derive(Debug, PartialEq, Eq)] -struct FlushedMsg; - -/// Sent to prematurely quit (before flushing). -#[derive(Debug, PartialEq, Eq)] -struct QuitMsg; - -/// Sent to prematurely quit (before flushing). -#[derive(Debug, PartialEq, Eq)] -enum InterruptMsg { - /// Switch to a mode where we drop messages if disconnected. - /// - /// Sending this before a flush ensures we won't get stuck trying to send - /// messages to a closed endpoint, but we will still send all messages to an open endpoint. - DropIfDisconnected, - - /// Quite immediately, dropping any unsent message. - Quit, -} - -enum MsgMsg { - LogMsg(LogMsg), - Flush, -} - -enum PacketMsg { - Packet(Vec), - Flush, -} - -/// Send [`LogMsg`]es to a server over TCP. -/// -/// The messages are encoded and sent on separate threads -/// so that calling [`Client::send`] is non-blocking. -pub struct Client { - msg_tx: Sender, - flushed_rx: Receiver, - encode_quit_tx: Sender, - send_quit_tx: Sender, - encode_join: Option>, - send_join: Option>, - - /// Only used for diagnostics, not for communication after `new()`. - addr: SocketAddr, -} - -impl Client { - /// Connect via TCP to this log server. - /// - /// `flush_timeout` is the minimum time the `TcpClient` will wait during a - /// flush before potentially dropping data. Note: Passing `None` here can - /// cause a call to `flush` to block indefinitely if a connection cannot be - /// established. - pub fn new(addr: SocketAddr, flush_timeout: Option) -> Self { - re_log::debug!("Connecting to remote {addr}…"); - - // TODO(emilk): keep track of how much memory is in each pipe - // and apply back-pressure to not use too much RAM. - let (msg_tx, msg_rx) = crossbeam::channel::unbounded(); - let (packet_tx, packet_rx) = crossbeam::channel::unbounded(); - let (flushed_tx, flushed_rx) = crossbeam::channel::unbounded(); - let (encode_quit_tx, encode_quit_rx) = crossbeam::channel::unbounded(); - let (send_quit_tx, send_quit_rx) = crossbeam::channel::unbounded(); - - // We don't compress the stream because we assume the SDK - // and server are on the same machine and compression - // can be expensive, see https://github.com/rerun-io/rerun/issues/2216 - let encoding_options = re_log_encoding::EncodingOptions::MSGPACK_UNCOMPRESSED; - - let encode_join = std::thread::Builder::new() - .name("msg_encoder".into()) - .spawn(move || { - msg_encode(encoding_options, &msg_rx, &encode_quit_rx, &packet_tx); - }) - .expect("Failed to spawn thread"); - - let send_join = std::thread::Builder::new() - .name("tcp_sender".into()) - .spawn(move || { - tcp_sender(addr, flush_timeout, &packet_rx, &send_quit_rx, &flushed_tx); - }) - .expect("Failed to spawn thread"); - - Self { - msg_tx, - flushed_rx, - encode_quit_tx, - send_quit_tx, - encode_join: Some(encode_join), - send_join: Some(send_join), - addr, - } - } - - pub fn send(&self, log_msg: LogMsg) { - self.send_msg_msg(MsgMsg::LogMsg(log_msg)); - } - - /// Stall until all messages so far has been sent. - pub fn flush(&self) { - re_log::debug!("Flushing message queue…"); - if self.msg_tx.send(MsgMsg::Flush).is_err() { - re_log::debug!("Flush failed: already shut down."); - return; - } - - match self.flushed_rx.recv() { - Ok(FlushedMsg) => { - re_log::debug!("Flush complete."); - } - Err(_) => { - // This can happen on Ctrl-C - re_log::warn!("Failed to flush pipeline - not all messages were sent."); - } - } - } - - /// Switch to a mode where we drop messages if disconnected. - /// - /// Calling this before a flush (or drop) ensures we won't get stuck trying to send - /// messages to a closed endpoint, but we will still send all messages to an open endpoint. - pub fn drop_if_disconnected(&self) { - self.send_quit_tx - .send(InterruptMsg::DropIfDisconnected) - .ok(); - } - - fn send_msg_msg(&self, msg: MsgMsg) { - // ignoring errors, because Ctrl-C can shut down the receiving end. - self.msg_tx.send(msg).ok(); - } -} - -impl Drop for Client { - /// Wait until everything has been sent. - fn drop(&mut self) { - re_log::debug!("Shutting down the client connection…"); - self.flush(); - // First shut down the encoder: - self.encode_quit_tx.send(QuitMsg).ok(); - self.encode_join.take().map(|j| j.join().ok()); - // Then the other threads: - self.send_quit_tx.send(InterruptMsg::Quit).ok(); - self.send_join.take().map(|j| j.join().ok()); - re_log::debug!("TCP client has shut down."); - } -} - -impl fmt::Debug for Client { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - // The other fields are all channels and join handles, so they are not usefully printable. - f.debug_struct("Client") - .field("addr", &self.addr) - .finish_non_exhaustive() - } -} - -fn msg_encode( - encoding_options: re_log_encoding::EncodingOptions, - msg_rx: &Receiver, - quit_rx: &Receiver, - packet_tx: &Sender, -) { - loop { - select! { - recv(msg_rx) -> msg_msg => { - let Ok(msg_msg) = msg_msg else { - re_log::debug!("Shutting down msg_encode thread: channel has closed"); - return; // channel has closed - }; - - let packet_msg = match &msg_msg { - MsgMsg::LogMsg(log_msg) => { - match re_log_encoding::encoder::encode_to_bytes( - re_build_info::CrateVersion::LOCAL, - encoding_options, std::iter::once(log_msg), - ) { - Ok(packet) => { - re_log::trace!("Encoded message of size {}", packet.len()); - Some(PacketMsg::Packet(packet)) - } - Err(err) => { - re_log::error_once!("Failed to encode log message: {err}"); - None - } - } - } - MsgMsg::Flush => Some(PacketMsg::Flush), - }; - - if let Some(packet_msg) = packet_msg { - if packet_tx.send(packet_msg).is_err() { - re_log::error!("Failed to send message to tcp_sender thread. Likely a shutdown race-condition."); - return; - } - } - } - recv(quit_rx) -> _quit_msg => { - re_log::debug!("Shutting down msg_encode thread: quit received"); - return; - } - } - } -} - -fn tcp_sender( - addr: SocketAddr, - flush_timeout: Option, - packet_rx: &Receiver, - quit_rx: &Receiver, - flushed_tx: &Sender, -) { - let mut tcp_client = crate::tcp_client::TcpClient::new(addr, flush_timeout); - // Once this flag has been set, we will drop all messages if the tcp_client is - // no longer connected. - let mut drop_if_disconnected = false; - - loop { - select! { - recv(packet_rx) -> packet_msg => { - if let Ok(packet_msg) = packet_msg { - match packet_msg { - PacketMsg::Packet(packet) => { - match send_until_success(&mut tcp_client, drop_if_disconnected, &packet, quit_rx) { - Some(InterruptMsg::Quit) => {return;} - Some(InterruptMsg::DropIfDisconnected) => { - drop_if_disconnected = true; - } - None => {} - } - } - PacketMsg::Flush => { - tcp_client.flush(); - flushed_tx - .send(FlushedMsg) - .expect("Main thread should still be alive"); - } - } - } else { - re_log::debug!("Shutting down tcp_sender thread: packet_rx channel has closed"); - return; // channel has closed - } - }, - recv(quit_rx) -> quit_msg => { match quit_msg { - // Don't terminate on receiving a `DropIfDisconnected`. It's a soft-quit that allows - // us to flush the pipeline. - Ok(InterruptMsg::DropIfDisconnected) => { - drop_if_disconnected = true; - } - Ok(InterruptMsg::Quit) => { - re_log::debug!("Shutting down tcp_sender thread: received Quit message"); - return; - } - Err(_) => { - re_log::debug!("Shutting down tcp_sender thread: quit_rx channel has closed"); - return; - } - }} - } - } -} - -fn send_until_success( - tcp_client: &mut crate::tcp_client::TcpClient, - drop_if_disconnected: bool, - packet: &[u8], - quit_rx: &Receiver, -) -> Option { - // Early exit if tcp_client is disconnected - if drop_if_disconnected && tcp_client.has_timed_out_for_flush() { - re_log::warn_once!("Dropping messages because tcp client has timed out."); - return None; - } - - if let Err(err) = tcp_client.send(packet) { - if drop_if_disconnected && tcp_client.has_timed_out_for_flush() { - re_log::warn_once!("Dropping messages because tcp client has timed out."); - return None; - } - // If this is the first time we fail to send the message, produce a warning. - re_log::debug!("Failed to send message: {err}"); - - let mut attempts = 1; - let mut sleep_ms = 100; - - loop { - select! { - recv(quit_rx) -> _quit_msg => { - re_log::warn_once!("Dropping messages because tcp client has timed out or quitting."); - return Some(_quit_msg.unwrap_or(InterruptMsg::Quit)); - } - default(std::time::Duration::from_millis(sleep_ms)) => { - if let Err(new_err) = tcp_client.send(packet) { - attempts += 1; - if attempts == 3 { - re_log::warn!("Failed to send message after {attempts} attempts: {err}"); - } - - if drop_if_disconnected && tcp_client.has_timed_out_for_flush() { - re_log::warn_once!("Dropping messages because tcp client has timed out."); - return None; - } - - const MAX_SLEEP_MS : u64 = 3000; - - sleep_ms = (sleep_ms * 2).min(MAX_SLEEP_MS); - - // Only produce subsequent warnings once we've saturated the back-off - if sleep_ms == MAX_SLEEP_MS && new_err.to_string() != err.to_string() { - re_log::warn!("Still failing to send message after {attempts} attempts: {err}"); - } - } else { - return None; - } - } - } - } - } else { - None - } -} diff --git a/crates/store/re_sdk_comms/src/lib.rs b/crates/store/re_sdk_comms/src/lib.rs deleted file mode 100644 index 7f74ff617f26..000000000000 --- a/crates/store/re_sdk_comms/src/lib.rs +++ /dev/null @@ -1,83 +0,0 @@ -//! TCP communications between a Rerun logging SDK and server/viewer. -//! -//! ## Feature flags -#![doc = document_features::document_features!()] -//! - -#[cfg(feature = "client")] -pub(crate) mod tcp_client; - -#[cfg(feature = "client")] -mod buffered_client; - -#[cfg(feature = "client")] -pub use {buffered_client::Client, tcp_client::ClientError}; - -#[cfg(feature = "server")] -mod server; - -#[cfg(feature = "server")] -pub use server::{serve, ServerError, ServerOptions}; - -/// Server connection error. -/// -/// This can only occur when using the `server` feature, -/// However it is defined here so that crates that want to react to this error can do so without -/// needing to depend on the `server` feature directly. -/// This is useful when processing errors from a passed-in `re_smart_channel` channel as done by `re_viewer` as of writing. -#[derive(thiserror::Error, Debug)] -pub enum ConnectionError { - #[error("An unknown client tried to connect")] - UnknownClient, - - #[error(transparent)] - VersionError(#[from] VersionError), - - #[error(transparent)] - SendError(#[from] std::io::Error), - - #[error(transparent)] - #[cfg(feature = "server")] - DecodeError(#[from] re_log_encoding::decoder::DecodeError), - - #[error("The receiving end of the channel was closed")] - ChannelDisconnected(#[from] re_smart_channel::SendError), -} - -#[derive(thiserror::Error, Debug)] -#[allow(unused)] -pub enum VersionError { - #[error("SDK client is using an older protocol version ({client_version}) than the SDK server ({server_version})")] - ClientIsOlder { - client_version: u16, - server_version: u16, - }, - - #[error("SDK client is using a newer protocol version ({client_version}) than the SDK server ({server_version})")] - ClientIsNewer { - client_version: u16, - server_version: u16, - }, -} - -pub const PROTOCOL_VERSION_0: u16 = 0; - -/// Added [`PROTOCOL_HEADER`]. Introduced for Rerun 0.16. -pub const PROTOCOL_VERSION_1: u16 = 1; - -/// Comes after version. -pub const PROTOCOL_HEADER: &str = "rerun"; - -pub const DEFAULT_SERVER_PORT: u16 = 9876; - -/// The default address of a Rerun TCP server which an SDK connects to. -pub fn default_server_addr() -> std::net::SocketAddr { - std::net::SocketAddr::from(([127, 0, 0, 1], DEFAULT_SERVER_PORT)) -} - -/// The default amount of time to wait for the TCP connection to resume during a flush -#[allow(clippy::unnecessary_wraps)] -pub fn default_flush_timeout() -> Option { - // NOTE: This is part of the SDK and meant to be used where we accept `Option` values. - Some(std::time::Duration::from_secs(2)) -} diff --git a/crates/store/re_sdk_comms/src/server.rs b/crates/store/re_sdk_comms/src/server.rs deleted file mode 100644 index 48a1c381c700..000000000000 --- a/crates/store/re_sdk_comms/src/server.rs +++ /dev/null @@ -1,412 +0,0 @@ -use std::{ - io::{ErrorKind, Read as _}, - net::{TcpListener, TcpStream}, - time::Instant, -}; - -use rand::{Rng as _, SeedableRng}; - -use re_log_types::{LogMsg, TimePoint, TimeType, TimelineName}; -use re_smart_channel::{Receiver, Sender}; - -use crate::{ConnectionError, VersionError}; - -#[derive(thiserror::Error, Debug)] -pub enum ServerError { - #[error("Failed to bind TCP address {bind_addr:?}. Another Rerun instance is probably running. {err}")] - TcpBindError { - bind_addr: String, - err: std::io::Error, - }, - - #[error(transparent)] - FailedToSpawnThread(#[from] std::io::Error), -} - -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct ServerOptions { - /// If the latency in the [`LogMsg`] channel is greater than this, - /// then start dropping messages in order to keep up. - pub max_latency_sec: f32, - - /// Turns `info`-level logs into `debug`-level logs. - pub quiet: bool, -} - -impl Default for ServerOptions { - fn default() -> Self { - Self { - max_latency_sec: f32::INFINITY, - quiet: false, - } - } -} - -/// Listen to multiple SDK:s connecting to us over TCP. -/// -/// ``` no_run -/// # use re_sdk_comms::{serve, ServerOptions}; -/// fn main() { -/// let log_msg_rx = serve("0.0.0.0", re_sdk_comms::DEFAULT_SERVER_PORT, ServerOptions::default()).unwrap(); -/// } -/// ``` -/// -/// Internally spawns a thread that listens for incoming TCP connections on the given `bind_ip` and `port` -/// and one thread per connected client. -// TODO(andreas): Reconsider if we should use `smol` tasks instead of threads both here and in re_ws_comms. -pub fn serve( - bind_ip: &str, - port: u16, - options: ServerOptions, -) -> Result, ServerError> { - let (tx, rx) = re_smart_channel::smart_channel( - // NOTE: We don't know until we start actually accepting clients! - re_smart_channel::SmartMessageSource::Unknown, - re_smart_channel::SmartChannelSource::TcpServer { port }, - ); - - let bind_addr = format!("{bind_ip}:{port}"); - let listener = TcpListener::bind(&bind_addr).map_err(|err| ServerError::TcpBindError { - bind_addr: bind_addr.clone(), - err, - })?; - - std::thread::Builder::new() - .name("rerun_sdk_comms: listener".to_owned()) - .spawn(move || { - listen_for_new_clients(&listener, options, &tx); - })?; - - if options.quiet { - re_log::debug!( - "Hosting a SDK server over TCP at {bind_addr}. Connect with the Rerun logging SDK." - ); - } else { - re_log::info!( - "Hosting a SDK server over TCP at {bind_addr}. Connect with the Rerun logging SDK." - ); - } - - Ok(rx) -} - -fn listen_for_new_clients(listener: &TcpListener, options: ServerOptions, tx: &Sender) { - // TODO(emilk): some way of aborting this loop - #[allow(clippy::infinite_loop)] - loop { - match listener.accept() { - Ok((stream, _)) => { - let addr = stream.peer_addr().ok(); - let tx = tx.clone_as(re_smart_channel::SmartMessageSource::TcpClient { addr }); - - std::thread::Builder::new() - .name("rerun_sdk_comms: client".to_owned()) - .spawn(move || { - spawn_client(stream, &tx, options, addr); - }) - .ok(); - } - Err(err) => { - if cfg!(target_os = "windows") { - // Windows error codes resolved to names via http://errorcodelookup.com/ - const WSANOTINITIALISED: i32 = 10093; - const WSAEINTR: i32 = 10004; - - if let Some(raw_os_error) = err.raw_os_error() { - #[allow(clippy::match_same_arms)] - match raw_os_error { - WSANOTINITIALISED => { - // This happens either if WSAStartup wasn't called beforehand, - // or WSACleanup was called as part of shutdown already. - // - // If we end up in here it's almost certainly the later case - // which implies that the process is shutting down. - break; - } - WSAEINTR => { - // A blocking operation was interrupted. - // This can only happen if the listener is closing, - // meaning that this server is shutting down. - break; - } - _ => {} - } - } - } - - re_log::warn!("Failed to accept incoming SDK client: {err}"); - } - } - } -} - -fn spawn_client( - stream: TcpStream, - tx: &Sender, - options: ServerOptions, - peer_addr: Option, -) { - let addr_string = peer_addr.map_or_else(|| "(unknown ip)".to_owned(), |addr| addr.to_string()); - - if let Err(err) = run_client(stream, &addr_string, tx, options) { - if let ConnectionError::SendError(err) = &err { - if err.kind() == ErrorKind::UnexpectedEof { - // Client gracefully severed the connection. - tx.quit(None).ok(); // best-effort at this point - return; - } - } - - if matches!(&err, ConnectionError::UnknownClient) { - // An unknown client that probably stumbled onto the wrong port. - // Don't log as an error (https://github.com/rerun-io/rerun/issues/5883). - re_log::debug!( - "Rejected incoming connection from unknown client at {addr_string}: {err}" - ); - } else { - re_log::warn_once!("Closing connection to client at {addr_string}: {err}"); - } - - let err: Box = err.into(); - tx.quit(Some(err)).ok(); // best-effort at this point - } -} - -fn run_client( - mut stream: TcpStream, - addr_string: &str, - tx: &Sender, - options: ServerOptions, -) -> Result<(), ConnectionError> { - #![allow(clippy::read_zero_byte_vec)] // false positive: https://github.com/rust-lang/rust-clippy/issues/9274 - - let mut client_version = [0_u8; 2]; - stream.read_exact(&mut client_version)?; - let client_version = u16::from_le_bytes(client_version); - - // The server goes into a backward compat mode - // if the client sends version 0 - if client_version == crate::PROTOCOL_VERSION_0 { - // Backwards compatibility mode: no protocol header, otherwise the same as version 1. - re_log::warn!("Client is using an old protocol version from before 0.16."); - } else { - // The protocol header was added in version 1 - let mut protocol_header = [0_u8; crate::PROTOCOL_HEADER.len()]; - stream.read_exact(&mut protocol_header)?; - - if std::str::from_utf8(&protocol_header) != Ok(crate::PROTOCOL_HEADER) { - return Err(ConnectionError::UnknownClient); - } - - if options.quiet { - re_log::debug!("New SDK client connected from: {addr_string}"); - } else { - re_log::info!("New SDK client connected from: {addr_string}"); - } - - let server_version = crate::PROTOCOL_VERSION_1; - match client_version.cmp(&server_version) { - std::cmp::Ordering::Less => { - return Err(ConnectionError::VersionError(VersionError::ClientIsOlder { - client_version, - server_version, - })); - } - std::cmp::Ordering::Equal => {} - std::cmp::Ordering::Greater => { - return Err(ConnectionError::VersionError(VersionError::ClientIsNewer { - client_version, - server_version, - })); - } - } - }; - - let mut congestion_manager = CongestionManager::new(options.max_latency_sec); - - let mut packet = Vec::new(); - - loop { - let mut packet_size = [0_u8; 4]; - stream.read_exact(&mut packet_size)?; - let packet_size = u32::from_le_bytes(packet_size); - - packet.resize(packet_size as usize, 0_u8); - stream.read_exact(&mut packet)?; - - re_log::trace!("Received packet of size {packet_size}."); - - congestion_manager.register_latency(tx.latency_sec()); - - let version_policy = re_log_encoding::VersionPolicy::Warn; - for msg in re_log_encoding::decoder::decode_bytes(version_policy, &packet)? { - if congestion_manager.should_send(&msg) { - tx.send(msg)?; - } else { - re_log::warn_once!( - "Input latency is over the max ({} s) - dropping packets.", - options.max_latency_sec - ); - } - } - } -} - -// ---------------------------------------------------------------------------- - -/// Decides how many messages to drop so that we achieve a desired maximum latency. -struct CongestionManager { - throttling: Throttling, - rng: rand::rngs::SmallRng, - timeline_histories: ahash::HashMap, -} - -#[derive(Default)] -struct TimelineThrottling { - chance_of_sending: f32, - send_time: std::collections::BTreeMap, -} - -impl CongestionManager { - pub fn new(max_latency_sec: f32) -> Self { - Self { - throttling: Throttling::new(max_latency_sec), - rng: rand::rngs::SmallRng::from_entropy(), - timeline_histories: Default::default(), - } - } - - pub fn register_latency(&mut self, latency_sec: f32) { - self.throttling.register_latency(latency_sec); - } - - pub fn should_send(&mut self, msg: &LogMsg) -> bool { - if self.throttling.accept_rate == 1.0 { - return true; // early out for common-case - } - - #[allow(clippy::match_same_arms)] - match msg { - // we don't want to drop any of these - LogMsg::SetStoreInfo(_) | LogMsg::BlueprintActivationCommand { .. } => true, - - LogMsg::ArrowMsg(_, arrow_msg) => self.should_send_time_point(&arrow_msg.timepoint_max), - } - } - - fn should_send_time_point(&mut self, time_point: &TimePoint) -> bool { - for (timeline, time) in time_point.iter() { - if timeline.typ() == TimeType::Sequence { - // We want to accept everything from the same sequence (e.g. frame nr) or nothing. - // See https://github.com/rerun-io/rerun/issues/430 for why. - return self.should_send_time(*timeline.name(), time.as_i64()); - } - } - - // There is no sequence timeline - just do stochastic filtering: - self.rng.gen::() < self.throttling.accept_rate - } - - fn should_send_time(&mut self, timeline: TimelineName, time: i64) -> bool { - let timeline_history = self.timeline_histories.entry(timeline).or_default(); - match timeline_history.send_time.entry(time) { - std::collections::btree_map::Entry::Vacant(entry) => { - // New time (e.g. frame nr)! Should we send messages in it? - // We use dithering via error diffusion to decide! - - let send_it = 0.5 < timeline_history.chance_of_sending; - entry.insert(send_it); - - if send_it { - // make it less likely we will send the next time: - timeline_history.chance_of_sending -= 1.0; - } else { - // make it more likely we will send it next time: - timeline_history.chance_of_sending += self.throttling.accept_rate; - } - - // Prune history so it doesn't grow too long. - // This only matters if messages arrive out-of-order. - // If we prune too much, we run the risk of taking a new (different) - // decision on a time we've previously seen, - // thus sending parts of a sequence-time instead of all-or-nothing. - while timeline_history.send_time.len() > 1024 { - let oldest_time = *timeline_history - .send_time - .keys() - .next() - .expect("safe because checked above"); - timeline_history.send_time.remove(&oldest_time); - } - - re_log::trace!("Send {timeline} {time}: {send_it}"); - - send_it - } - std::collections::btree_map::Entry::Occupied(entry) => { - *entry.get() // Reuse previous decision - } - } - } -} - -// ---------------------------------------------------------------------------- - -/// Figures out how large fraction of messages to send based on -/// the current latency vs our desired max latency. -struct Throttling { - max_latency_sec: f32, - accept_rate: f32, - last_time: Instant, - last_log_time: Instant, -} - -impl Throttling { - pub fn new(max_latency_sec: f32) -> Self { - Self { - max_latency_sec, - accept_rate: 1.0, - last_time: Instant::now(), - last_log_time: Instant::now(), - } - } - - pub fn register_latency(&mut self, current_latency: f32) { - let now = Instant::now(); - let dt = (now - self.last_time).as_secs_f32(); - self.last_time = now; - - let is_good = current_latency < self.max_latency_sec; - - if is_good && self.accept_rate == 1.0 { - return; // early out - } - - /// If we let it go too low, we won't accept any messages, - /// and then we won't ever recover. - const MIN_ACCEPT_RATE: f32 = 0.01; - - // This is quite ad-hoc, but better than nothing. - // Perhaps it's worth investigating a more rigorous additive increase/multiplicative decrease congestion protocol. - if is_good { - // Slowly improve our accept-rate, slower the closer we are: - let goodness = (self.max_latency_sec - current_latency) / self.max_latency_sec; - self.accept_rate += goodness * dt / 25.0; - } else { - // Quickly decrease our accept-rate, quicker the worse we are: - let badness = (current_latency - self.max_latency_sec) / self.max_latency_sec; - let badness = badness.clamp(0.5, 2.0); - self.accept_rate -= badness * dt / 5.0; - } - - self.accept_rate = self.accept_rate.clamp(MIN_ACCEPT_RATE, 1.0); - - if self.last_log_time.elapsed().as_secs_f32() > 1.0 { - re_log::debug!( - "Currently dropping {:.2}% of messages to keep latency low", - 100.0 * (1.0 - self.accept_rate) - ); - self.last_log_time = Instant::now(); - } - } -} diff --git a/crates/store/re_sdk_comms/src/tcp_client.rs b/crates/store/re_sdk_comms/src/tcp_client.rs deleted file mode 100644 index cd8e51ffe254..000000000000 --- a/crates/store/re_sdk_comms/src/tcp_client.rs +++ /dev/null @@ -1,196 +0,0 @@ -use std::{ - io::Write, - net::{SocketAddr, TcpStream}, - time::{Duration, Instant}, -}; - -#[derive(thiserror::Error, Debug)] -pub enum ClientError { - #[error("Failed to connect to Rerun server at {addr:?}: {err}")] - Connect { - addr: SocketAddr, - err: std::io::Error, - }, - - #[error("Failed to send to Rerun server at {addr:?}: {err}")] - Send { - addr: SocketAddr, - err: std::io::Error, - }, -} - -/// State of the [`TcpStream`] -/// -/// Because the [`TcpClient`] lazily connects on [`TcpClient::send`], it needs a -/// very simple state machine to track the state of the connection. -/// [`TcpStreamState::Pending`] is the nominal state for any new TCP connection -/// when we successfully connect, we transition to [`TcpStreamState::Connected`]. -enum TcpStreamState { - /// The [`TcpStream`] is yet to be connected. - /// - /// Tracks the duration and connection attempts since the last time the client - /// was `Connected.` - /// - /// Behavior: Try to connect on next [`TcpClient::connect`] or [`TcpClient::send`]. - /// - /// Transitions: - /// - Pending -> Connected on successful connection. - /// - Pending -> Pending on failed connection. - Pending { - start_time: Instant, - num_attempts: usize, - }, - - /// A healthy [`TcpStream`] ready to send packets - /// - /// Behavior: Send packets on [`TcpClient::send`] - /// - /// Transitions: - /// - Connected -> Pending on send error - Connected(TcpStream), -} - -impl TcpStreamState { - fn reset() -> Self { - Self::Pending { - start_time: Instant::now(), - num_attempts: 0, - } - } -} - -/// Connect to a rerun server and send log messages. -/// -/// Blocking connection. -pub struct TcpClient { - addr: SocketAddr, - stream_state: TcpStreamState, - flush_timeout: Option, -} - -impl TcpClient { - pub fn new(addr: SocketAddr, flush_timeout: Option) -> Self { - Self { - addr, - stream_state: TcpStreamState::reset(), - flush_timeout, - } - } - - /// Returns `false` on failure. Does nothing if already connected. - /// - /// [`Self::send`] will call this. - pub fn connect(&mut self) -> Result<(), ClientError> { - match self.stream_state { - TcpStreamState::Connected(_) => Ok(()), - TcpStreamState::Pending { - start_time, - num_attempts, - } => { - re_log::debug!("Connecting to {:?}…", self.addr); - let timeout = std::time::Duration::from_secs(5); - match TcpStream::connect_timeout(&self.addr, timeout) { - Ok(mut stream) => { - re_log::debug!("Connected to {:?}.", self.addr); - - if let Err(err) = stream - .write(&crate::PROTOCOL_VERSION_1.to_le_bytes()) - .and_then(|_| stream.write(crate::PROTOCOL_HEADER.as_bytes())) - { - self.stream_state = TcpStreamState::Pending { - start_time, - num_attempts: num_attempts + 1, - }; - Err(ClientError::Send { - addr: self.addr, - err, - }) - } else { - self.stream_state = TcpStreamState::Connected(stream); - Ok(()) - } - } - Err(err) => { - self.stream_state = TcpStreamState::Pending { - start_time, - num_attempts: num_attempts + 1, - }; - Err(ClientError::Connect { - addr: self.addr, - err, - }) - } - } - } - } - } - - /// Blocks until it is sent. - pub fn send(&mut self, packet: &[u8]) -> Result<(), ClientError> { - use std::io::Write as _; - - self.connect()?; - - if let TcpStreamState::Connected(stream) = &mut self.stream_state { - re_log::trace!("Sending a packet of size {}…", packet.len()); - if let Err(err) = stream.write(&(packet.len() as u32).to_le_bytes()) { - self.stream_state = TcpStreamState::reset(); - return Err(ClientError::Send { - addr: self.addr, - err, - }); - } - - if let Err(err) = stream.write(packet) { - self.stream_state = TcpStreamState::reset(); - return Err(ClientError::Send { - addr: self.addr, - err, - }); - } - - Ok(()) - } else { - unreachable!("self.connect should have ensured this"); - } - } - - /// Wait until all logged data have been sent. - pub fn flush(&mut self) { - re_log::trace!("Attempting to flush TCP stream…"); - match &mut self.stream_state { - TcpStreamState::Pending { .. } => { - re_log::warn_once!( - "Tried to flush while TCP stream was still Pending. Data was possibly dropped." - ); - } - TcpStreamState::Connected(stream) => { - if let Err(err) = stream.flush() { - re_log::warn!("Failed to flush TCP stream: {err}"); - self.stream_state = TcpStreamState::reset(); - } else { - re_log::trace!("TCP stream flushed."); - } - } - } - } - - /// Check if the underlying [`TcpStream`] is in the [`TcpStreamState::Pending`] state - /// and has reached the flush timeout threshold. - /// - /// Note that this only occurs after a failure to connect or a failure to send. - pub fn has_timed_out_for_flush(&self) -> bool { - match self.stream_state { - TcpStreamState::Pending { - start_time, - num_attempts, - } => { - // If a timeout wasn't provided, never timeout - self.flush_timeout.is_some_and(|timeout| { - Instant::now().duration_since(start_time) > timeout && num_attempts > 0 - }) - } - TcpStreamState::Connected(_) => false, - } - } -} From c972194c60b84aeaafc880ab0d3dc09105ec0133 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 29 Jan 2025 13:08:30 +0100 Subject: [PATCH 39/87] Update link checker config --- lychee.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lychee.toml b/lychee.toml index 2ffa5759535a..3c8256ec50bf 100644 --- a/lychee.toml +++ b/lychee.toml @@ -67,7 +67,7 @@ exclude_path = [ "scripts/lint.py", # Contains url-matching regexes that aren't actual urls "scripts/screenshot_compare/assets/templates/", "crates/viewer/re_viewer/src/reflection/mod.rs", # Checker struggles how links from examples are escaped here. They are all checked elsewhere, so not an issue. - "crates/store/re_grpc_client/src/address.rs", # Contains some malformed URLs, but they are not actual links. + "crates/store/re_grpc_client/src/redap/address.rs", # Contains some malformed URLs, but they are not actual links. ] # Exclude URLs and mail addresses from checking (supports regex). From 3f2296bfccf0b4953ffbee183dbbde6fd5249add Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 29 Jan 2025 13:10:17 +0100 Subject: [PATCH 40/87] Fix python lints --- rerun_py/rerun_sdk/rerun/sinks.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/rerun_py/rerun_sdk/rerun/sinks.py b/rerun_py/rerun_sdk/rerun/sinks.py index bda63cd7f1b3..8dab4f6bb8ce 100644 --- a/rerun_py/rerun_sdk/rerun/sinks.py +++ b/rerun_py/rerun_sdk/rerun/sinks.py @@ -47,7 +47,11 @@ def connect( See also: [`rerun.init`][], [`rerun.set_global_data_recording`][]. """ - return connect_grpc(url, default_blueprint=default_blueprint, recording=recording) + return connect_grpc( + url, + default_blueprint=default_blueprint, + recording=recording, # NOLINT: conversion not needed + ) def connect_grpc( @@ -452,7 +456,7 @@ def spawn( memory_limit=memory_limit, hide_welcome_screen=hide_welcome_screen, default_blueprint=default_blueprint, - recording=recording, + recording=recording, # NOLINT: conversion not needed ) From ac0bf39444f9b105388fa935a778c5b5e6d4098e Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 29 Jan 2025 13:12:46 +0100 Subject: [PATCH 41/87] Update script helpers to pass http url instead of plain addr --- rerun_py/rerun_sdk/rerun/blueprint/api.py | 2 +- rerun_py/rerun_sdk/rerun/script_helpers.py | 7 +++---- scripts/run_all.py | 2 +- scripts/run_python_e2e_test.py | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/rerun_py/rerun_sdk/rerun/blueprint/api.py b/rerun_py/rerun_sdk/rerun/blueprint/api.py index 263f59624d20..30a8b2c2243f 100644 --- a/rerun_py/rerun_sdk/rerun/blueprint/api.py +++ b/rerun_py/rerun_sdk/rerun/blueprint/api.py @@ -686,7 +686,7 @@ def spawn( """ _spawn_viewer(port=port, memory_limit=memory_limit, hide_welcome_screen=hide_welcome_screen) - self.connect(application_id=application_id, addr=f"127.0.0.1:{port}") + self.connect(application_id=application_id, url=f"http://127.0.0.1:{port}") def spawn_grpc( self, application_id: str, port: int = 9876, memory_limit: str = "75%", hide_welcome_screen: bool = False diff --git a/rerun_py/rerun_sdk/rerun/script_helpers.py b/rerun_py/rerun_sdk/rerun/script_helpers.py index f3e5f26b9305..015795717f18 100644 --- a/rerun_py/rerun_sdk/rerun/script_helpers.py +++ b/rerun_py/rerun_sdk/rerun/script_helpers.py @@ -51,7 +51,7 @@ def script_add_args(parser: ArgumentParser) -> None: action="store_true", help="Serve a web viewer (WARNING: experimental feature)", ) - parser.add_argument("--addr", type=str, default=None, help="Connect to this ip:port") + parser.add_argument("--url", type=str, default=None, help="Connect to this HTTP(S) URL") parser.add_argument("--save", type=str, default=None, help="Save data to a .rrd file at this path") parser.add_argument( "-o", @@ -111,9 +111,8 @@ def script_setup( rec.serve(default_blueprint=default_blueprint) # type: ignore[attr-defined] elif args.connect: # Send logging data to separate `rerun` process. - # You can omit the argument to connect to the default address, - # which is `127.0.0.1:9876`. - rec.connect(args.addr, default_blueprint=default_blueprint) # type: ignore[attr-defined] + # You can omit the argument to connect to the default URL. + rec.connect(args.url, default_blueprint=default_blueprint) # type: ignore[attr-defined] elif args.save is not None: rec.save(args.save, default_blueprint=default_blueprint) # type: ignore[attr-defined] elif not args.headless: diff --git a/scripts/run_all.py b/scripts/run_all.py index 5b15aa80b3ff..fc955c2b6db1 100755 --- a/scripts/run_all.py +++ b/scripts/run_all.py @@ -86,7 +86,7 @@ def run_py_example(path: str, viewer_port: int | None = None, *, wait: bool = Tr if save is not None: args += [f"--save={save}"] if viewer_port is not None: - args += ["--connect", f"--addr=127.0.0.1:{viewer_port}"] + args += ["--connect", f"--url=http://127.0.0.1:{viewer_port}"] return start_process( args, diff --git a/scripts/run_python_e2e_test.py b/scripts/run_python_e2e_test.py index f437522c6ae1..19975f80f4c3 100755 --- a/scripts/run_python_e2e_test.py +++ b/scripts/run_python_e2e_test.py @@ -85,7 +85,7 @@ def run_example(example: str, extra_args: list[str]) -> None: rerun_process = subprocess.Popen(cmd, env=env) time.sleep(0.5) # Wait for rerun server to start to remove a logged warning - cmd = ["python", "-m", example, "--connect", "--addr", f"127.0.0.1:{PORT}"] + extra_args + cmd = ["python", "-m", example, "--connect", "--url", f"http://127.0.0.1:{PORT}"] + extra_args python_process = subprocess.Popen(cmd, env=env) print("Waiting for python process to finish…") From 510ccd6e896e74923236603a7d582ed92e10bf59 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 29 Jan 2025 13:15:26 +0100 Subject: [PATCH 42/87] Add `MessageEvent` web-sys feature --- crates/store/re_log_encoding/Cargo.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/store/re_log_encoding/Cargo.toml b/crates/store/re_log_encoding/Cargo.toml index ac0bb865f1a0..2213a1365444 100644 --- a/crates/store/re_log_encoding/Cargo.toml +++ b/crates/store/re_log_encoding/Cargo.toml @@ -77,7 +77,10 @@ web-time = { workspace = true, optional = true } js-sys = { workspace = true, optional = true } wasm-bindgen = { workspace = true, optional = true } wasm-bindgen-futures = { workspace = true, optional = true } -web-sys = { workspace = true, optional = true, features = ["Window"] } +web-sys = { workspace = true, optional = true, features = [ + "Window", + "MessageEvent", +] } [dev-dependencies] re_types.workspace = true From 70c6e95d3b368419934e02c6a7eae1cbcd395db8 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 29 Jan 2025 13:19:33 +0100 Subject: [PATCH 43/87] Remove `WebSocket` endpoint category --- crates/viewer/re_viewer/src/web_tools.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/crates/viewer/re_viewer/src/web_tools.rs b/crates/viewer/re_viewer/src/web_tools.rs index ade38ace9158..b7b4dc32c48c 100644 --- a/crates/viewer/re_viewer/src/web_tools.rs +++ b/crates/viewer/re_viewer/src/web_tools.rs @@ -1,6 +1,5 @@ //! Web-specific tools used by various parts of the application. -use anyhow::Context as _; use re_log::ResultExt; use serde::Deserialize; use std::{ops::ControlFlow, sync::Arc}; @@ -88,9 +87,6 @@ enum EndpointCategory { /// gRPC Rerun Data Platform URL, e.g. `rerun://ip:port/recording/1234` RerunGrpc(String), - /// A remote Rerun server. - WebSocket(String), - /// An eventListener for rrd posted from containing html WebEventListener(String), @@ -104,8 +100,6 @@ impl EndpointCategory { Self::HttpRrd(uri) } else if uri.starts_with("rerun://") { Self::RerunGrpc(uri) - } else if uri.starts_with("ws:") || uri.starts_with("wss:") { - Self::WebSocket(uri) } else if uri.starts_with("web_event:") { Self::WebEventListener(uri) } else if uri.starts_with("temp:") { @@ -186,8 +180,6 @@ pub fn url_to_receiver( })); Ok(rx) } - EndpointCategory::WebSocket(url) => re_data_source::connect_to_ws_url(&url, Some(ui_waker)) - .with_context(|| format!("Failed to connect to WebSocket server at {url}.")), EndpointCategory::MessageProxy(url) => { re_grpc_client::message_proxy::read::stream(&url, Some(ui_waker)) From e1cb37d4c0325b8b966cbb2abdac8dc387122685 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 29 Jan 2025 13:20:04 +0100 Subject: [PATCH 44/87] Format --- lychee.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lychee.toml b/lychee.toml index 3c8256ec50bf..6ee47a9adf95 100644 --- a/lychee.toml +++ b/lychee.toml @@ -63,11 +63,11 @@ exclude_path = [ "venv", # Actually ignored files beyond .gitignore - "crates/utils/re_analytics/src/event.rs", # Contains test with malformed urls - "scripts/lint.py", # Contains url-matching regexes that aren't actual urls + "crates/utils/re_analytics/src/event.rs", # Contains test with malformed urls + "scripts/lint.py", # Contains url-matching regexes that aren't actual urls "scripts/screenshot_compare/assets/templates/", - "crates/viewer/re_viewer/src/reflection/mod.rs", # Checker struggles how links from examples are escaped here. They are all checked elsewhere, so not an issue. - "crates/store/re_grpc_client/src/redap/address.rs", # Contains some malformed URLs, but they are not actual links. + "crates/viewer/re_viewer/src/reflection/mod.rs", # Checker struggles how links from examples are escaped here. They are all checked elsewhere, so not an issue. + "crates/store/re_grpc_client/src/redap/address.rs", # Contains some malformed URLs, but they are not actual links. ] # Exclude URLs and mail addresses from checking (supports regex). From 10710b2cf084bb2420b9624ec37e73b69490d1ff Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 29 Jan 2025 13:31:37 +0100 Subject: [PATCH 45/87] Fix some rust lints --- crates/store/re_grpc_client/src/message_proxy/read.rs | 5 ++++- crates/store/re_grpc_server/src/shutdown.rs | 2 +- crates/top/re_sdk/src/log_sink.rs | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/store/re_grpc_client/src/message_proxy/read.rs b/crates/store/re_grpc_client/src/message_proxy/read.rs index 6deb558c355f..24e3bd601611 100644 --- a/crates/store/re_grpc_client/src/message_proxy/read.rs +++ b/crates/store/re_grpc_client/src/message_proxy/read.rs @@ -170,7 +170,10 @@ mod tests { ($name:ident, $url:literal, expected: $expected_http:literal) => { #[test] fn $name() { - assert_eq!(MessageProxyUrl::parse($url).to_http(), $expected_http); + assert_eq!( + MessageProxyUrl::parse($url).map(|v| v.to_http()), + Ok($expected_http) + ); } }; } diff --git a/crates/store/re_grpc_server/src/shutdown.rs b/crates/store/re_grpc_server/src/shutdown.rs index 77dda02230c6..743c8c7864e3 100644 --- a/crates/store/re_grpc_server/src/shutdown.rs +++ b/crates/store/re_grpc_server/src/shutdown.rs @@ -28,7 +28,7 @@ pub struct Shutdown(Option>); impl Shutdown { /// Returns a future that resolves when the signal is sent. /// - /// If this was constructed with [`never`], then it never resolves. + /// If this was constructed with [`never()`], then it never resolves. pub async fn wait(self) { if let Some(rx) = self.0 { rx.await.ok(); diff --git a/crates/top/re_sdk/src/log_sink.rs b/crates/top/re_sdk/src/log_sink.rs index 893af45e005a..063bfea86d83 100644 --- a/crates/top/re_sdk/src/log_sink.rs +++ b/crates/top/re_sdk/src/log_sink.rs @@ -37,7 +37,7 @@ pub trait LogSink: Send + Sync + 'static { fn flush_blocking(&self); /// Drops all pending data currently sitting in the sink's send buffers if it is unable to - /// flush it for any reason (e.g. a broken TCP connection for a [`TcpSink`]). + /// flush it for any reason. #[inline] fn drop_if_disconnected(&self) {} From 5b87b549dd8a626369fe771ceddec80dd8901772 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 29 Jan 2025 14:23:22 +0100 Subject: [PATCH 46/87] Remove `WsClient` and `TcpServer` channel sources --- crates/utils/re_smart_channel/src/lib.rs | 38 ------------------- .../utils/re_smart_channel/src/receive_set.rs | 17 +++++++-- crates/viewer/re_viewer/src/app.rs | 11 ++---- crates/viewer/re_viewer/src/app_state.rs | 2 - .../re_viewer/src/ui/recordings_panel.rs | 6 +-- crates/viewer/re_viewer/src/ui/top_panel.rs | 14 +------ .../re_viewer/src/viewer_analytics/event.rs | 3 +- .../viewer/re_viewer_context/src/store_hub.rs | 10 ++++- 8 files changed, 29 insertions(+), 72 deletions(-) diff --git a/crates/utils/re_smart_channel/src/lib.rs b/crates/utils/re_smart_channel/src/lib.rs index f678a7719239..61a6f04a772a 100644 --- a/crates/utils/re_smart_channel/src/lib.rs +++ b/crates/utils/re_smart_channel/src/lib.rs @@ -46,20 +46,6 @@ pub enum SmartChannelSource { /// process. Sdk, - /// The channel was created in the context of fetching data from a Rerun WebSocket server. - /// - /// We are likely running in a web browser. - WsClient { - /// The server we are connected to (or are trying to connect to) - ws_server_url: String, - }, - - /// The channel was created in the context of receiving data from one or more Rerun SDKs - /// over TCP. - /// - /// We are a TCP server listening on this port. - TcpServer { port: u16 }, - /// The channel was created in the context of streaming in RRD data from standard input. Stdin, @@ -85,8 +71,6 @@ impl std::fmt::Display for SmartChannelSource { Self::RrdWebEventListener => "Web event listener".fmt(f), Self::JsChannel { channel_name } => write!(f, "Javascript channel: {channel_name}"), Self::Sdk => "SDK".fmt(f), - Self::WsClient { ws_server_url } => ws_server_url.fmt(f), - Self::TcpServer { port } => write!(f, "TCP server, port {port}"), Self::MessageProxy { url } => write!(f, "gRPC server: {url}"), Self::Stdin => "Standard input".fmt(f), } @@ -98,9 +82,7 @@ impl SmartChannelSource { match self { Self::File(_) | Self::Sdk | Self::RrdWebEventListener | Self::Stdin => false, Self::RrdHttpStream { .. } - | Self::WsClient { .. } | Self::JsChannel { .. } - | Self::TcpServer { .. } | Self::RerunGrpcStream { .. } | Self::MessageProxy { .. } => true, } @@ -141,21 +123,6 @@ pub enum SmartMessageSource { /// The sender is a Rerun SDK running from another thread in the same process. Sdk, - /// The sender is a WebSocket client fetching data from a Rerun WebSocket server. - /// - /// We are likely running in a web browser. - WsClient { - /// The server we are connected to (or are trying to connect to) - ws_server_url: String, - }, - - /// The sender is a TCP client. - TcpClient { - // NOTE: Optional as we might not be able to retrieve the peer's address for some obscure - // reason. - addr: Option, - }, - /// The data is streaming in from standard input. Stdin, @@ -184,11 +151,6 @@ impl std::fmt::Display for SmartMessageSource { Self::RrdWebEventCallback => "web_callback".into(), Self::JsChannelPush => "javascript".into(), Self::Sdk => "sdk".into(), - Self::WsClient { ws_server_url } => ws_server_url.clone(), - Self::TcpClient { addr } => format!( - "tcp://{}", - addr.map_or_else(|| "(unknown ip)".to_owned(), |addr| addr.to_string()) - ), Self::Stdin => "stdin".into(), }) } diff --git a/crates/utils/re_smart_channel/src/receive_set.rs b/crates/utils/re_smart_channel/src/receive_set.rs index 21b21c2dbc7f..bc10117f0859 100644 --- a/crates/utils/re_smart_channel/src/receive_set.rs +++ b/crates/utils/re_smart_channel/src/receive_set.rs @@ -53,7 +53,16 @@ impl ReceiveSet { // - aren't network sources // - don't point at the given `uri` SmartChannelSource::RrdHttpStream { url, .. } => url != uri, - SmartChannelSource::WsClient { ws_server_url } => ws_server_url != uri, + SmartChannelSource::MessageProxy { url } => { + fn strip_prefix(s: &str) -> &str { + // TODO(#8761): URL prefix + s.strip_prefix("http") + .or_else(|| s.strip_prefix("temp")) + .unwrap_or(s) + } + + strip_prefix(url) != strip_prefix(uri) + } _ => true, }); } @@ -75,12 +84,12 @@ impl ReceiveSet { !self.is_empty() } - /// Does this viewer accept inbound TCP connections? - pub fn accepts_tcp_connections(&self) -> bool { + /// Does this viewer accept inbound gRPC connections? + pub fn accepts_grpc_connections(&self) -> bool { re_tracing::profile_function!(); self.sources() .iter() - .any(|s| matches!(**s, SmartChannelSource::TcpServer { .. })) + .any(|s| matches!(**s, SmartChannelSource::MessageProxy { .. })) } /// No connected receivers? diff --git a/crates/viewer/re_viewer/src/app.rs b/crates/viewer/re_viewer/src/app.rs index c44334375158..0ce22016b80d 100644 --- a/crates/viewer/re_viewer/src/app.rs +++ b/crates/viewer/re_viewer/src/app.rs @@ -541,12 +541,10 @@ impl App { | SmartChannelSource::RrdHttpStream { .. } | SmartChannelSource::RerunGrpcStream { .. } => false, - SmartChannelSource::WsClient { .. } - | SmartChannelSource::JsChannel { .. } + SmartChannelSource::JsChannel { .. } | SmartChannelSource::RrdWebEventListener | SmartChannelSource::Sdk | SmartChannelSource::MessageProxy { .. } - | SmartChannelSource::TcpServer { .. } | SmartChannelSource::Stdin => true, }); } @@ -1561,15 +1559,14 @@ impl App { | SmartChannelSource::Stdin | SmartChannelSource::RrdWebEventListener | SmartChannelSource::Sdk - | SmartChannelSource::WsClient { .. } | SmartChannelSource::JsChannel { .. } => { return true; // We expect data soon, so fade-in } - SmartChannelSource::TcpServer { .. } | SmartChannelSource::MessageProxy { .. } => { - // We start a TCP server by default in native rerun, i.e. when just running `rerun`, + SmartChannelSource::MessageProxy { .. } => { + // We start a gRPC server by default in native rerun, i.e. when just running `rerun`, // and in that case fading in the welcome screen would be slightly annoying. - // However, we also use the TCP server for sending data from the logging SDKs + // However, we also use the gRPC server for sending data from the logging SDKs // when they call `spawn()`, and in that case we really want to fade in the welcome screen. // Therefore `spawn()` uses the special `--expect-data-soon` flag // (handled earlier in this function), so here we know we are in the other case: diff --git a/crates/viewer/re_viewer/src/app_state.rs b/crates/viewer/re_viewer/src/app_state.rs index dde57820d4cb..709524e99e4e 100644 --- a/crates/viewer/re_viewer/src/app_state.rs +++ b/crates/viewer/re_viewer/src/app_state.rs @@ -648,8 +648,6 @@ fn recording_config_entry<'cfgs>( // Live data - follow it! re_smart_channel::SmartChannelSource::RrdHttpStream { follow: true, .. } | re_smart_channel::SmartChannelSource::Sdk - | re_smart_channel::SmartChannelSource::WsClient { .. } - | re_smart_channel::SmartChannelSource::TcpServer { .. } | re_smart_channel::SmartChannelSource::MessageProxy { .. } | re_smart_channel::SmartChannelSource::Stdin | re_smart_channel::SmartChannelSource::JsChannel { .. } => PlayState::Following, diff --git a/crates/viewer/re_viewer/src/ui/recordings_panel.rs b/crates/viewer/re_viewer/src/ui/recordings_panel.rs index af6528560615..311c8ca8ac56 100644 --- a/crates/viewer/re_viewer/src/ui/recordings_panel.rs +++ b/crates/viewer/re_viewer/src/ui/recordings_panel.rs @@ -65,8 +65,6 @@ fn loading_receivers_ui(ctx: &ViewerContext<'_>, rx: &ReceiveSet, ui: &m SmartChannelSource::RrdWebEventListener | SmartChannelSource::JsChannel { .. } | SmartChannelSource::Sdk - | SmartChannelSource::WsClient { .. } - | SmartChannelSource::TcpServer { .. } | re_smart_channel::SmartChannelSource::MessageProxy { .. } | SmartChannelSource::Stdin => { // These show up in the top panel - see `top_panel.rs`. @@ -92,9 +90,7 @@ fn loading_receivers_ui(ctx: &ViewerContext<'_>, rx: &ReceiveSet, ui: &m resp }), ); - if let SmartChannelSource::TcpServer { .. } | SmartChannelSource::MessageProxy { .. } = - source.as_ref() - { + if let SmartChannelSource::MessageProxy { .. } = source.as_ref() { response.on_hover_text("You can connect to this viewer from a Rerun SDK"); } } diff --git a/crates/viewer/re_viewer/src/ui/top_panel.rs b/crates/viewer/re_viewer/src/ui/top_panel.rs index e390b75528f1..64dcd2930d18 100644 --- a/crates/viewer/re_viewer/src/ui/top_panel.rs +++ b/crates/viewer/re_viewer/src/ui/top_panel.rs @@ -167,8 +167,6 @@ fn connection_status_ui(ui: &mut egui::Ui, rx: &ReceiveSet SmartChannelSource::RrdWebEventListener | SmartChannelSource::Sdk - | SmartChannelSource::WsClient { .. } - | SmartChannelSource::TcpServer { .. } | SmartChannelSource::MessageProxy { .. } | SmartChannelSource::JsChannel { .. } => true, } @@ -203,10 +201,9 @@ fn connection_status_ui(ui: &mut egui::Ui, rx: &ReceiveSet | SmartChannelSource::RerunGrpcStream { .. } | SmartChannelSource::RrdWebEventListener | SmartChannelSource::JsChannel { .. } - | SmartChannelSource::Sdk - | SmartChannelSource::WsClient { .. } => None, + | SmartChannelSource::Sdk => None, - SmartChannelSource::TcpServer { .. } | SmartChannelSource::MessageProxy { .. } => { + SmartChannelSource::MessageProxy { .. } => { Some("Waiting for an SDK to connect".to_owned()) } }; @@ -236,13 +233,6 @@ fn connection_status_ui(ui: &mut egui::Ui, rx: &ReceiveSet re_smart_channel::SmartChannelSource::Sdk => { "Waiting for logging data from SDK".to_owned() } - re_smart_channel::SmartChannelSource::WsClient { ws_server_url } => { - // TODO(emilk): it would be even better to know whether or not we are connected, or are attempting to connect - format!("Waiting for data from {ws_server_url}") - } - re_smart_channel::SmartChannelSource::TcpServer { port } => { - format!("Listening on TCP port {port}") - } } } } diff --git a/crates/viewer/re_viewer/src/viewer_analytics/event.rs b/crates/viewer/re_viewer/src/viewer_analytics/event.rs index fcf456941923..109cf4c270fc 100644 --- a/crates/viewer/re_viewer/src/viewer_analytics/event.rs +++ b/crates/viewer/re_viewer/src/viewer_analytics/event.rs @@ -142,12 +142,11 @@ pub fn open_recording( re_smart_channel::SmartChannelSource::File(_) => "file", // .rrd, .png, .glb, … re_smart_channel::SmartChannelSource::RrdHttpStream { .. } => "http", re_smart_channel::SmartChannelSource::RerunGrpcStream { .. } => "grpc", + // vvv spawn(), connect() vvv re_smart_channel::SmartChannelSource::MessageProxy { .. } => "temp", // TODO(#8761): URL prefix re_smart_channel::SmartChannelSource::RrdWebEventListener { .. } => "web_event", re_smart_channel::SmartChannelSource::JsChannel { .. } => "javascript", // mediated via rerun-js re_smart_channel::SmartChannelSource::Sdk => "sdk", // show() - re_smart_channel::SmartChannelSource::WsClient { .. } => "ws_client", // spawn() - re_smart_channel::SmartChannelSource::TcpServer { .. } => "tcp_server", // connect() re_smart_channel::SmartChannelSource::Stdin => "stdin", }); diff --git a/crates/viewer/re_viewer_context/src/store_hub.rs b/crates/viewer/re_viewer_context/src/store_hub.rs index bc01b1c98c29..ea35f932ee97 100644 --- a/crates/viewer/re_viewer_context/src/store_hub.rs +++ b/crates/viewer/re_viewer_context/src/store_hub.rs @@ -695,8 +695,14 @@ impl StoreHub { // - don't point at the given `uri` match data_source { re_smart_channel::SmartChannelSource::RrdHttpStream { url, .. } => url != uri, - re_smart_channel::SmartChannelSource::WsClient { ws_server_url } => { - ws_server_url != uri + re_smart_channel::SmartChannelSource::MessageProxy { url } => { + fn strip_prefix(s: &str) -> &str { + s.strip_prefix("http") + .or_else(|| s.strip_prefix("temp")) + .unwrap_or(s) + } + + strip_prefix(url) != strip_prefix(uri) } _ => true, } From bf08d1d86343207c81ea26aebe945070a7b8c2ed Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 29 Jan 2025 14:23:36 +0100 Subject: [PATCH 47/87] Update some docs to mention `gRPC` instead of `TCP` --- crates/top/re_sdk/src/lib.rs | 2 +- crates/top/re_sdk/src/recording_stream.rs | 6 +++--- crates/top/rerun-cli/README.md | 2 +- crates/top/rerun/README.md | 2 +- crates/top/rerun/src/commands/entrypoint.rs | 11 ++++------- .../re_viewer/data/quick_start_guides/cpp_connect.md | 2 +- .../re_viewer/data/quick_start_guides/cpp_spawn.md | 2 +- .../data/quick_start_guides/how_does_it_work.md | 4 ++-- .../data/quick_start_guides/quick_start_connect.cpp | 4 ++-- .../data/quick_start_guides/quick_start_connect.rs | 2 +- crates/viewer/re_viewer_context/src/item.rs | 2 +- 11 files changed, 18 insertions(+), 21 deletions(-) diff --git a/crates/top/re_sdk/src/lib.rs b/crates/top/re_sdk/src/lib.rs index ebee8848540b..ad7242bc0482 100644 --- a/crates/top/re_sdk/src/lib.rs +++ b/crates/top/re_sdk/src/lib.rs @@ -59,7 +59,7 @@ impl crate::sink::LogSink for re_log_encoding::FileSink { /// Different destinations for log messages. /// /// This is how you select whether the log stream ends up -/// sent over TCP, written to file, etc. +/// sent over gRPC, written to file, etc. pub mod sink { pub use crate::binary_stream_sink::{ BinaryStreamSink, BinaryStreamSinkError, BinaryStreamStorage, diff --git a/crates/top/re_sdk/src/recording_stream.rs b/crates/top/re_sdk/src/recording_stream.rs index 7270b47b398c..09f5c18419b8 100644 --- a/crates/top/re_sdk/src/recording_stream.rs +++ b/crates/top/re_sdk/src/recording_stream.rs @@ -444,7 +444,7 @@ impl RecordingStreamBuilder { } /// Spawns a new Rerun Viewer process from an executable available in PATH, then creates a new - /// [`RecordingStream`] that is pre-configured to stream the data through to that viewer over TCP. + /// [`RecordingStream`] that is pre-configured to stream the data through to that viewer over gRPC. /// /// If a Rerun Viewer is already listening on this port, the stream will be redirected to /// that viewer instead of starting a new one. @@ -463,7 +463,7 @@ impl RecordingStreamBuilder { } /// Spawns a new Rerun Viewer process from an executable available in PATH, then creates a new - /// [`RecordingStream`] that is pre-configured to stream the data through to that viewer over TCP. + /// [`RecordingStream`] that is pre-configured to stream the data through to that viewer over gRPC. /// /// If a Rerun Viewer is already listening on this port, the stream will be redirected to /// that viewer instead of starting a new one. @@ -1566,7 +1566,7 @@ impl RecordingStream { /// /// ## Data loss /// - /// If the current sink is in a broken state (e.g. a TCP sink with a broken connection that + /// If the current sink is in a broken state (e.g. a gRPC sink with a broken connection that /// cannot be repaired), all pending data in its buffers will be dropped. pub fn set_sink(&self, sink: Box) { if self.is_forked_child() { diff --git a/crates/top/rerun-cli/README.md b/crates/top/rerun-cli/README.md index c48ab0f2f51f..e86b54a2f3ab 100644 --- a/crates/top/rerun-cli/README.md +++ b/crates/top/rerun-cli/README.md @@ -20,7 +20,7 @@ Alternatively, you may skip enabling the `nasm` feature, but this may result in The `rerun` CLI can act either as a server, a viewer, or both, depending on which options you use when you start it. -Running `rerun` with no arguments will start the viewer, waiting for an SDK to connect to it over TCP. +Running `rerun` with no arguments will start the viewer, waiting for an SDK to connect to it over gRPC. Run `rerun --help` for more. diff --git a/crates/top/rerun/README.md b/crates/top/rerun/README.md index f5b3513cbb96..105b2ea140c7 100644 --- a/crates/top/rerun/README.md +++ b/crates/top/rerun/README.md @@ -48,7 +48,7 @@ Alternatively, you may skip enabling the `nasm` feature, but this may result in The `rerun` CLI can act either as a server, a viewer, or both, depending on which options you use when you start it. -Running `rerun` with no arguments will start the viewer, waiting for an SDK to connect to it over TCP. +Running `rerun` with no arguments will start the viewer, waiting for an SDK to connect to it over gRPC. Run `rerun --help` for more. diff --git a/crates/top/rerun/src/commands/entrypoint.rs b/crates/top/rerun/src/commands/entrypoint.rs index 96a1aba14a78..3b5c81a134e8 100644 --- a/crates/top/rerun/src/commands/entrypoint.rs +++ b/crates/top/rerun/src/commands/entrypoint.rs @@ -23,7 +23,7 @@ use crate::commands::AnalyticsCommands; const LONG_ABOUT: &str = r#" The Rerun command-line interface: * Spawn viewers to visualize Rerun recordings and other supported formats. -* Start TCP and WebSocket servers to share recordings over the network, on native or web. +* Start a gRPC server to share recordings over the network, on native or web. * Inspect, edit and filter Rerun recordings. "#; @@ -56,7 +56,7 @@ Examples: Open an .rrd file and stream it to a Web Viewer: rerun recording.rrd --web-viewer - Host a Rerun TCP server which listens for incoming TCP connections from the logging SDK, buffer the log messages, and serves the results over WebSockets: + Host a Rerun gRPC server which listens for incoming connections from the logging SDK, buffer the log messages, and serves the results: rerun --serve-web Host a Rerun Server which serves a recording over WebSocket to any connecting Rerun Viewers: @@ -150,12 +150,9 @@ When persisted, the state will be stored at the following locations: #[clap(long)] serve: bool, - /// Serve the recordings over WebSocket to one or more Rerun Viewers. + /// This will host a web-viewer over HTTP, and a gRPC server. /// - /// This will also host a web-viewer over HTTP that can connect to the WebSocket address, - /// but you can also connect with the native binary. - /// - /// `rerun --serve-web` will act like a proxy, listening for incoming TCP connection from + /// The server will act like a proxy, listening for incoming connections from /// logging SDKs, and forwarding it to Rerun viewers. #[clap(long)] serve_web: bool, diff --git a/crates/viewer/re_viewer/data/quick_start_guides/cpp_connect.md b/crates/viewer/re_viewer/data/quick_start_guides/cpp_connect.md index e27546ddeb9d..aadbe1f10dc6 100644 --- a/crates/viewer/re_viewer/data/quick_start_guides/cpp_connect.md +++ b/crates/viewer/re_viewer/data/quick_start_guides/cpp_connect.md @@ -1,7 +1,7 @@ # C++ quick start ## Installing the Rerun Viewer -The Rerun C++ SDK works by connecting to an awaiting Rerun Viewer over TCP. +The Rerun C++ SDK works by connecting to an awaiting Rerun Viewer over gRPC. If you need to install the viewer, follow the [installation guide](https://www.rerun.io/docs/getting-started/installing-viewer). Two of the more common ways to install the Rerun are: * Via cargo: `cargo install rerun-cli --locked --features nasm` (see note below) diff --git a/crates/viewer/re_viewer/data/quick_start_guides/cpp_spawn.md b/crates/viewer/re_viewer/data/quick_start_guides/cpp_spawn.md index 231c8f419b0f..b102c88f9667 100644 --- a/crates/viewer/re_viewer/data/quick_start_guides/cpp_spawn.md +++ b/crates/viewer/re_viewer/data/quick_start_guides/cpp_spawn.md @@ -1,7 +1,7 @@ # C++ quick start ## Installing the Rerun Viewer -The Rerun C++ SDK works by connecting to an awaiting Rerun Viewer over TCP. +The Rerun C++ SDK works by connecting to an awaiting Rerun Viewer over gRPC. If you need to install the viewer, follow the [installation guide](https://www.rerun.io/docs/getting-started/installing-viewer). Two of the more common ways to install the Rerun are: * Via cargo: `cargo install rerun-cli --locked --features nasm` (see note below) diff --git a/crates/viewer/re_viewer/data/quick_start_guides/how_does_it_work.md b/crates/viewer/re_viewer/data/quick_start_guides/how_does_it_work.md index ef9a43f90b95..b19be13e7509 100644 --- a/crates/viewer/re_viewer/data/quick_start_guides/how_does_it_work.md +++ b/crates/viewer/re_viewer/data/quick_start_guides/how_does_it_work.md @@ -2,8 +2,8 @@ Rerun's goal is to make handling and visualizing multimodal data streams easy and performant. -Rerun is made of two main building blocks: the SDK and the Viewer. The data provided by the user code is serialized by the SDK and transferred (via a log file, a TCP socket, a WebSocket, etc.) to the Viewer process for visualization. You can learn more about Rerun's operating modes [here](https://www.rerun.io/docs/reference/sdk/operating-modes). +Rerun is made of two main building blocks: the SDK and the Viewer. The data provided by the user code is serialized by the SDK and transferred (via a log file, a gRPC connection, etc.) to the Viewer process for visualization. You can learn more about Rerun's operating modes [here](https://www.rerun.io/docs/reference/sdk/operating-modes). -In the example above, the SDK connects via a TCP socket to the viewer. +In the example above, the SDK connects via a gRPC connection to the viewer. The `log()` function logs _entities_ represented by the "entity path" provided as first argument. Entities are a collection of _components_, which hold the actual data such as position, color, or pixel data. _Archetypes_ such as `Points3D` are builder objects which help creating entities with a consistent set of components that are recognized by the Viewer (they can be entirely bypassed when required by advanced use-cases). You can learn more about Rerun's data model [here](https://www.rerun.io/docs/concepts/entity-component). diff --git a/crates/viewer/re_viewer/data/quick_start_guides/quick_start_connect.cpp b/crates/viewer/re_viewer/data/quick_start_guides/quick_start_connect.cpp index 733c63c2aafa..41733b22fc03 100644 --- a/crates/viewer/re_viewer/data/quick_start_guides/quick_start_connect.cpp +++ b/crates/viewer/re_viewer/data/quick_start_guides/quick_start_connect.cpp @@ -4,9 +4,9 @@ using namespace rerun::demo; int main() { - // Create a new `RecordingStream` which sends data over TCP to the viewer process. + // Create a new `RecordingStream` which sends data over gRPC to the viewer process. const auto rec = rerun::RecordingStream("rerun_example_quick_start_connect"); - rec.connect_grpc().exit_on_failure(); + rec.connect().exit_on_failure(); // Create some data using the `grid` utility function. std::vector points = grid3d(-10.f, 10.f, 10); diff --git a/crates/viewer/re_viewer/data/quick_start_guides/quick_start_connect.rs b/crates/viewer/re_viewer/data/quick_start_guides/quick_start_connect.rs index 0ccd3064e122..be638bda8c76 100644 --- a/crates/viewer/re_viewer/data/quick_start_guides/quick_start_connect.rs +++ b/crates/viewer/re_viewer/data/quick_start_guides/quick_start_connect.rs @@ -3,7 +3,7 @@ use rerun::{demo_util::grid, external::glam}; fn main() -> Result<(), Box> { - // Create a new `RecordingStream` which sends data over TCP to the viewer process. + // Create a new `RecordingStream` which sends data over gRPC to the viewer process. let rec = rerun::RecordingStreamBuilder::new("rerun_example_quick_start_connect").connect()?; // Create some data using the `grid` utility function. diff --git a/crates/viewer/re_viewer_context/src/item.rs b/crates/viewer/re_viewer_context/src/item.rs index 6dec4f817629..7caf9fdf9b43 100644 --- a/crates/viewer/re_viewer_context/src/item.rs +++ b/crates/viewer/re_viewer_context/src/item.rs @@ -12,7 +12,7 @@ pub enum Item { /// Select a specific application, to see which recordings and blueprints are loaded for it. AppId(re_log_types::ApplicationId), - /// A place where data comes from, e.g. the path to a .rrd or a TCP port. + /// A place where data comes from, e.g. the path to a .rrd or a gRPC URL. DataSource(re_smart_channel::SmartChannelSource), /// A recording (or blueprint) From d0f1dfd4f900c1bfb75e2caa3083c7c06210de48 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 29 Jan 2025 15:50:01 +0100 Subject: [PATCH 48/87] Fix message proxy client not responding to flush during connect --- .../re_grpc_client/src/message_proxy/read.rs | 45 ++++++------ .../re_grpc_client/src/message_proxy/write.rs | 70 ++++++++++++++++--- .../quick_start_guides/quick_start_spawn.cpp | 2 +- rerun_cpp/tests/recording_stream.cpp | 39 +++++------ 4 files changed, 102 insertions(+), 54 deletions(-) diff --git a/crates/store/re_grpc_client/src/message_proxy/read.rs b/crates/store/re_grpc_client/src/message_proxy/read.rs index 24e3bd601611..88706b02eff0 100644 --- a/crates/store/re_grpc_client/src/message_proxy/read.rs +++ b/crates/store/re_grpc_client/src/message_proxy/read.rs @@ -34,39 +34,38 @@ pub fn stream( } /// Represents a URL to a gRPC server. +#[derive(Debug, Clone, PartialEq, Eq)] pub struct MessageProxyUrl(String); impl MessageProxyUrl { /// Parses as a regular URL, the protocol must be `temp://`, `http://`, or `https://`. pub fn parse(url: &str) -> Result { - if let Some(url) = url.strip_prefix("http") { + if url.starts_with("http") { let _ = Url::parse(url).map_err(|err| InvalidMessageProxyUrl { url: url.to_owned(), msg: err.to_string(), })?; - return Ok(Self(url.to_owned())); + Ok(Self(url.to_owned())) } + // TODO(#8761): URL prefix + else if let Some(url) = url.strip_prefix("temp") { + let url = format!("http{url}"); - let Some(url) = url.strip_prefix("temp") else { - let scheme = url.split_once("://").map(|(a, _)| a).ok_or("unknown"); - return Err(InvalidMessageProxyUrl { - url: url.to_owned(), - msg: format!( - "Invalid scheme {scheme:?}, expected {:?}", - // TODO(#8761): URL prefix - "temp" - ), - }); - }; - let url = format!("http{url}"); + let _ = Url::parse(&url).map_err(|err| InvalidMessageProxyUrl { + url: url.clone(), + msg: err.to_string(), + })?; - let _ = Url::parse(&url).map_err(|err| InvalidMessageProxyUrl { - url: url.clone(), - msg: err.to_string(), - })?; + Ok(Self(url)) + } else { + let scheme = url.split_once("://").map(|(a, _)| a).ok_or("unknown"); - Ok(Self(url)) + Err(InvalidMessageProxyUrl { + url: url.to_owned(), + msg: format!("Invalid scheme {scheme:?}, expected {:?}", "temp"), + }) + } } pub fn to_http(&self) -> String { @@ -88,7 +87,7 @@ impl std::str::FromStr for MessageProxyUrl { } } -#[derive(Debug, thiserror::Error)] +#[derive(Debug, thiserror::Error, PartialEq, Eq)] #[error("invalid message proxy url {url:?}: {msg}")] pub struct InvalidMessageProxyUrl { pub url: String, @@ -172,12 +171,14 @@ mod tests { fn $name() { assert_eq!( MessageProxyUrl::parse($url).map(|v| v.to_http()), - Ok($expected_http) + Ok($expected_http.to_owned()) ); } }; } - test_parse_url!(basic, "temp://127.0.0.1:1852", expected: "http://127.0.0.1:1852"); + test_parse_url!(basic_temp, "temp://127.0.0.1:1852", expected: "http://127.0.0.1:1852"); + // TODO(#8761): URL prefix + test_parse_url!(basic_http, "http://127.0.0.1:1852", expected: "http://127.0.0.1:1852"); test_parse_url!(invalid, "definitely not valid", error); } diff --git a/crates/store/re_grpc_client/src/message_proxy/write.rs b/crates/store/re_grpc_client/src/message_proxy/write.rs index 5c23aad6725f..26333f01247c 100644 --- a/crates/store/re_grpc_client/src/message_proxy/write.rs +++ b/crates/store/re_grpc_client/src/message_proxy/write.rs @@ -1,5 +1,6 @@ use std::thread; use std::thread::JoinHandle; +use std::time::Duration; use re_log_encoding::Compression; use re_log_types::LogMsg; @@ -96,10 +97,18 @@ impl Drop for Client { re_log::debug!("Shutting down message proxy client"); // Wait for flush self.flush(); - // Quit immediately after that - no messages are left in the channel - self.shutdown_tx.try_send(()).ok(); + + // Quit immediately - no more messages left in the queue + if let Err(err) = self.shutdown_tx.try_send(()) { + re_log::error!("failed to gracefully shut down message proxy client: {err}"); + return; + }; + // Wait for the shutdown - self.thread.take().map(|t| t.join().ok()); + if let Some(thread) = self.thread.take() { + thread.join().ok(); + }; + re_log::debug!("Message proxy client has shut down"); } } @@ -117,16 +126,59 @@ async fn message_proxy_client( return; } }; - let channel = match endpoint.connect().await { - Ok(channel) => channel, - Err(err) => { - re_log::error!("Failed to connect to message proxy server: {err}"); - return; + + // Temporarily buffer messages while we're connecting: + let mut buffered_messages = vec![]; + let channel = loop { + match endpoint.connect().await { + Ok(channel) => break channel, + Err(err) => { + re_log::debug!("failed to connect to message proxy server: {err}"); + tokio::select! { + cmd = cmd_rx.recv() => { + match cmd { + Some(Cmd::LogMsg(msg)) => { + buffered_messages.push(msg); + } + Some(Cmd::Flush(tx)) => { + re_log::debug!("Flush requested"); + if tx.send(()).is_err() { + re_log::debug!("Failed to respond to flush: channel is closed"); + return; + }; + } + None => { + re_log::debug!("Channel closed"); + return; + } + } + } + _ = shutdown_rx.recv() => { + re_log::debug!("shutting down client without flush"); + return; + } + _ = tokio::time::sleep(Duration::from_millis(200)) => { + continue; + } + } + } } }; let mut client = MessageProxyClient::new(channel); let stream = async_stream::stream! { + for msg in buffered_messages { + let msg = match re_log_encoding::protobuf_conversions::log_msg_to_proto(msg, compression) { + Ok(msg) => msg, + Err(err) => { + re_log::error!("Failed to encode message: {err}"); + break; + } + }; + + yield msg; + } + loop { tokio::select! { cmd = cmd_rx.recv() => { @@ -161,7 +213,7 @@ async fn message_proxy_client( } _ = shutdown_rx.recv() => { - re_log::debug!("Shutting down without flush"); + re_log::debug!("Shutting down client without flush"); return; } } diff --git a/crates/viewer/re_viewer/data/quick_start_guides/quick_start_spawn.cpp b/crates/viewer/re_viewer/data/quick_start_guides/quick_start_spawn.cpp index 559f7c9c4615..c05f8ef5167a 100644 --- a/crates/viewer/re_viewer/data/quick_start_guides/quick_start_spawn.cpp +++ b/crates/viewer/re_viewer/data/quick_start_guides/quick_start_spawn.cpp @@ -4,7 +4,7 @@ using namespace rerun::demo; int main() { - // Create a new `RecordingStream` which sends data over TCP to the viewer process. + // Create a new `RecordingStream` which sends data over gRPC to the viewer process. const auto rec = rerun::RecordingStream("rerun_example_quick_start_spawn"); rec.spawn().exit_on_failure(); diff --git a/rerun_cpp/tests/recording_stream.cpp b/rerun_cpp/tests/recording_stream.cpp index 87dd5b913b8e..f2cb2a5dbe1d 100644 --- a/rerun_cpp/tests/recording_stream.cpp +++ b/rerun_cpp/tests/recording_stream.cpp @@ -264,13 +264,13 @@ SCENARIO("RecordingStream can be used for logging archetypes and components", TE THEN("an archetype can be logged") { stream.log( "log_archetype-splat", - rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}} - ).with_colors(rerun::Color(0xFF0000FF)) + rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}}) + .with_colors(rerun::Color(0xFF0000FF)) ); stream.log_static( "log_archetype-splat", - rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}} - ).with_colors(rerun::Color(0xFF0000FF)) + rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}}) + .with_colors(rerun::Color(0xFF0000FF)) ); } @@ -367,27 +367,26 @@ SCENARIO("RecordingStream can log to file", TEST_TAG) { } } -void test_logging_to_connection(const char* address, const rerun::RecordingStream& stream) { +void test_logging_to_connection(const char* url, const rerun::RecordingStream& stream) { // We changed to taking std::string_view instead of const char* and constructing such from nullptr crashes // at least on some C++ implementations. // If we'd want to support this in earnest we'd have to create out own string_view type. // - // AND_GIVEN("a nullptr for the socket address") { + // AND_GIVEN("a nullptr for the socket url") { // THEN("then the connect call returns a null argument error") { // CHECK(stream.connect(nullptr, 0.0f).code == rerun::ErrorCode::UnexpectedNullArgument); // } // } - AND_GIVEN("an invalid address for the socket address") { + AND_GIVEN("an invalid url") { THEN("then the save call fails") { CHECK( - stream.connect_grpc("definitely not valid!").code == - rerun::ErrorCode::InvalidServerUrl + stream.connect("definitely not valid!").code == rerun::ErrorCode::InvalidServerUrl ); } } - AND_GIVEN("a valid socket address " << address) { + AND_GIVEN("a valid url " << url) { THEN("save call with zero timeout returns no error") { - REQUIRE(stream.connect_grpc(address).is_ok()); + CHECK(stream.connect(url).code == rerun::ErrorCode::Ok); WHEN("logging a component and then flushing") { check_logged_error([&] { @@ -427,10 +426,10 @@ void test_logging_to_connection(const char* address, const rerun::RecordingStrea } SCENARIO("RecordingStream can connect", TEST_TAG) { - const char* address = "127.0.0.1:9876"; + const char* url = "http://127.0.0.1:1852"; GIVEN("a new RecordingStream") { rerun::RecordingStream stream("test-local"); - test_logging_to_connection(address, stream); + test_logging_to_connection(url, stream); } WHEN("setting a global RecordingStream and then discarding it") { { @@ -438,7 +437,7 @@ SCENARIO("RecordingStream can connect", TEST_TAG) { stream.set_global(); } GIVEN("the current recording stream") { - test_logging_to_connection(address, rerun::RecordingStream::current()); + test_logging_to_connection(url, rerun::RecordingStream::current()); } } } @@ -491,17 +490,13 @@ SCENARIO("Recording stream handles serialization failure during logging graceful THEN("calling log with an array logs the serialization error") { check_logged_error( - [&] { - stream.log(path, std::array{component, component}); - }, + [&] { stream.log(path, std::array{component, component}); }, expected_error.code ); } THEN("calling log with a vector logs the serialization error") { check_logged_error( - [&] { - stream.log(path, std::vector{component, component}); - }, + [&] { stream.log(path, std::vector{component, component}); }, expected_error.code ); } @@ -643,8 +638,8 @@ SCENARIO("Deprecated log_static still works", TEST_TAG) { THEN("an archetype can be logged") { stream.log_static( "log_archetype-splat", - rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}} - ).with_colors(rerun::Color(0xFF0000FF)) + rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}}) + .with_colors(rerun::Color(0xFF0000FF)) ); } } From eb7505b736da49ce81dd9e7110da6547150ba187 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 29 Jan 2025 16:36:42 +0100 Subject: [PATCH 49/87] Run `cpp-fmt` --- rerun_cpp/tests/recording_stream.cpp | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/rerun_cpp/tests/recording_stream.cpp b/rerun_cpp/tests/recording_stream.cpp index f2cb2a5dbe1d..0b9d19a43fae 100644 --- a/rerun_cpp/tests/recording_stream.cpp +++ b/rerun_cpp/tests/recording_stream.cpp @@ -264,13 +264,13 @@ SCENARIO("RecordingStream can be used for logging archetypes and components", TE THEN("an archetype can be logged") { stream.log( "log_archetype-splat", - rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}}) - .with_colors(rerun::Color(0xFF0000FF)) + rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}} + ).with_colors(rerun::Color(0xFF0000FF)) ); stream.log_static( "log_archetype-splat", - rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}}) - .with_colors(rerun::Color(0xFF0000FF)) + rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}} + ).with_colors(rerun::Color(0xFF0000FF)) ); } @@ -490,13 +490,17 @@ SCENARIO("Recording stream handles serialization failure during logging graceful THEN("calling log with an array logs the serialization error") { check_logged_error( - [&] { stream.log(path, std::array{component, component}); }, + [&] { + stream.log(path, std::array{component, component}); + }, expected_error.code ); } THEN("calling log with a vector logs the serialization error") { check_logged_error( - [&] { stream.log(path, std::vector{component, component}); }, + [&] { + stream.log(path, std::vector{component, component}); + }, expected_error.code ); } @@ -638,8 +642,8 @@ SCENARIO("Deprecated log_static still works", TEST_TAG) { THEN("an archetype can be logged") { stream.log_static( "log_archetype-splat", - rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}}) - .with_colors(rerun::Color(0xFF0000FF)) + rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}} + ).with_colors(rerun::Color(0xFF0000FF)) ); } } From 571ed2fc4e4a2e16750ce07bd2f1d8406a4e8641 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Wed, 29 Jan 2025 18:53:56 +0100 Subject: [PATCH 50/87] Back to 9876 as the default port --- crates/store/re_grpc_client/src/message_proxy/read.rs | 4 ++-- crates/store/re_grpc_server/src/lib.rs | 2 +- crates/top/re_sdk/src/recording_stream.rs | 4 ++-- docs/snippets/all/concepts/app-model/native-sync.py | 2 +- examples/rust/custom_callback/src/app.rs | 2 +- examples/rust/custom_callback/src/viewer.rs | 2 +- examples/rust/custom_view/src/main.rs | 2 +- examples/rust/extend_viewer_ui/src/main.rs | 2 +- rerun_cpp/src/rerun/recording_stream.hpp | 4 ++-- rerun_cpp/tests/recording_stream.cpp | 2 +- rerun_py/rerun_sdk/rerun/sinks.py | 6 +++--- 11 files changed, 16 insertions(+), 16 deletions(-) diff --git a/crates/store/re_grpc_client/src/message_proxy/read.rs b/crates/store/re_grpc_client/src/message_proxy/read.rs index 88706b02eff0..c9b0405edb4f 100644 --- a/crates/store/re_grpc_client/src/message_proxy/read.rs +++ b/crates/store/re_grpc_client/src/message_proxy/read.rs @@ -177,8 +177,8 @@ mod tests { }; } - test_parse_url!(basic_temp, "temp://127.0.0.1:1852", expected: "http://127.0.0.1:1852"); + test_parse_url!(basic_temp, "temp://127.0.0.1:9876", expected: "http://127.0.0.1:9876"); // TODO(#8761): URL prefix - test_parse_url!(basic_http, "http://127.0.0.1:1852", expected: "http://127.0.0.1:1852"); + test_parse_url!(basic_http, "http://127.0.0.1:9876", expected: "http://127.0.0.1:9876"); test_parse_url!(invalid, "definitely not valid", error); } diff --git a/crates/store/re_grpc_server/src/lib.rs b/crates/store/re_grpc_server/src/lib.rs index 0885261b3d34..8e35b594702e 100644 --- a/crates/store/re_grpc_server/src/lib.rs +++ b/crates/store/re_grpc_server/src/lib.rs @@ -23,7 +23,7 @@ use tonic::transport::server::TcpIncoming; use tonic::transport::Server; use tower_http::cors::CorsLayer; -pub const DEFAULT_SERVER_PORT: u16 = 1852; +pub const DEFAULT_SERVER_PORT: u16 = 9876; pub const DEFAULT_MEMORY_LIMIT: MemoryLimit = MemoryLimit::UNLIMITED; /// Listen for incoming clients on `addr`. diff --git a/crates/top/re_sdk/src/recording_stream.rs b/crates/top/re_sdk/src/recording_stream.rs index 09f5c18419b8..dfed098132fc 100644 --- a/crates/top/re_sdk/src/recording_stream.rs +++ b/crates/top/re_sdk/src/recording_stream.rs @@ -327,7 +327,7 @@ impl RecordingStreamBuilder { /// /// ```no_run /// let rec = re_sdk::RecordingStreamBuilder::new("rerun_example_app") - /// .connect_opts("http://127.0.0.1:1852")?; + /// .connect_opts("http://127.0.0.1:9876")?; /// # Ok::<(), Box>(()) /// ``` pub fn connect_opts(self, url: impl Into) -> RecordingStreamResult { @@ -359,7 +359,7 @@ impl RecordingStreamBuilder { /// /// ```no_run /// let rec = re_sdk::RecordingStreamBuilder::new("rerun_example_app") - /// .connect_grpc_opts("http://127.0.0.1:1852")?; + /// .connect_grpc_opts("http://127.0.0.1:9876")?; /// # Ok::<(), Box>(()) /// ``` pub fn connect_grpc_opts( diff --git a/docs/snippets/all/concepts/app-model/native-sync.py b/docs/snippets/all/concepts/app-model/native-sync.py index cefb40529346..d66ab4cd2f99 100755 --- a/docs/snippets/all/concepts/app-model/native-sync.py +++ b/docs/snippets/all/concepts/app-model/native-sync.py @@ -4,7 +4,7 @@ rr.init("rerun_example_native_sync") -# Connect to the Rerun gRPC server using the default address and url: http://localhost:1852 +# Connect to the Rerun gRPC server using the default address and url: http://localhost:9876 rr.connect_grpc() # Log data as usual, thereby pushing it into the TCP socket. diff --git a/examples/rust/custom_callback/src/app.rs b/examples/rust/custom_callback/src/app.rs index aaee7e3991ea..99ce71f7a9af 100644 --- a/examples/rust/custom_callback/src/app.rs +++ b/examples/rust/custom_callback/src/app.rs @@ -15,7 +15,7 @@ use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; async fn main() -> Result<(), Box> { let mut app = ControlApp::bind("127.0.0.1:8888").await?.run(); let rec = rerun::RecordingStreamBuilder::new("rerun_example_custom_callback") - .connect_opts("http://127.0.0.1:1853")?; + .connect_opts("http://127.0.0.1:9877")?; // Add a handler for incoming messages let add_rec = rec.clone(); diff --git a/examples/rust/custom_callback/src/viewer.rs b/examples/rust/custom_callback/src/viewer.rs index 06fe61fd24b9..975d87276263 100644 --- a/examples/rust/custom_callback/src/viewer.rs +++ b/examples/rust/custom_callback/src/viewer.rs @@ -25,7 +25,7 @@ async fn main() -> Result<(), Box> { // Listen for gRPC connections from Rerun's logging SDKs. // There are other ways of "feeding" the viewer though - all you need is a `re_smart_channel::Receiver`. let rx = re_grpc_server::spawn_with_recv( - "0.0.0.0:1853".parse()?, + "0.0.0.0:9877".parse()?, "75%".parse()?, re_grpc_server::shutdown::never(), ); diff --git a/examples/rust/custom_view/src/main.rs b/examples/rust/custom_view/src/main.rs index 0fd9d385fd7a..34f9690f4583 100644 --- a/examples/rust/custom_view/src/main.rs +++ b/examples/rust/custom_view/src/main.rs @@ -26,7 +26,7 @@ async fn main() -> Result<(), Box> { // Listen for gRPC connections from Rerun's logging SDKs. // There are other ways of "feeding" the viewer though - all you need is a `re_smart_channel::Receiver`. let rx = re_grpc_server::spawn_with_recv( - "0.0.0.0:1852".parse()?, + "0.0.0.0:9876".parse()?, "75%".parse()?, re_grpc_server::shutdown::never(), ); diff --git a/examples/rust/extend_viewer_ui/src/main.rs b/examples/rust/extend_viewer_ui/src/main.rs index 8b65784a856f..7f8a39f37a7f 100644 --- a/examples/rust/extend_viewer_ui/src/main.rs +++ b/examples/rust/extend_viewer_ui/src/main.rs @@ -25,7 +25,7 @@ async fn main() -> Result<(), Box> { // Listen for gRPC connections from Rerun's logging SDKs. // There are other ways of "feeding" the viewer though - all you need is a `re_smart_channel::Receiver`. let rx = re_grpc_server::spawn_with_recv( - "0.0.0.0:1852".parse()?, + "0.0.0.0:9876".parse()?, "75%".parse()?, re_grpc_server::shutdown::never(), ); diff --git a/rerun_cpp/src/rerun/recording_stream.hpp b/rerun_cpp/src/rerun/recording_stream.hpp index 7dde4d4e790a..afee81616e94 100644 --- a/rerun_cpp/src/rerun/recording_stream.hpp +++ b/rerun_cpp/src/rerun/recording_stream.hpp @@ -138,14 +138,14 @@ namespace rerun { /// Requires that you first start a Rerun Viewer by typing 'rerun' in a terminal. /// /// This function returns immediately. - Error connect(std::string_view url = "http://127.0.0.1:1852") const; + Error connect(std::string_view url = "http://127.0.0.1:9876") const; /// Connect to a remote Rerun Viewer on the given HTTP(S) URL. /// /// Requires that you first start a Rerun Viewer by typing 'rerun' in a terminal. /// /// This function returns immediately. - Error connect_grpc(std::string_view url = "http://127.0.0.1:1852") const; + Error connect_grpc(std::string_view url = "http://127.0.0.1:9876") const; /// Spawns a new Rerun Viewer process from an executable available in PATH, then connects to it /// over gRPC. diff --git a/rerun_cpp/tests/recording_stream.cpp b/rerun_cpp/tests/recording_stream.cpp index 0b9d19a43fae..9b1ac9492728 100644 --- a/rerun_cpp/tests/recording_stream.cpp +++ b/rerun_cpp/tests/recording_stream.cpp @@ -426,7 +426,7 @@ void test_logging_to_connection(const char* url, const rerun::RecordingStream& s } SCENARIO("RecordingStream can connect", TEST_TAG) { - const char* url = "http://127.0.0.1:1852"; + const char* url = "http://127.0.0.1:9876"; GIVEN("a new RecordingStream") { rerun::RecordingStream stream("test-local"); test_logging_to_connection(url, stream); diff --git a/rerun_py/rerun_sdk/rerun/sinks.py b/rerun_py/rerun_sdk/rerun/sinks.py index 8dab4f6bb8ce..c13cd884a15a 100644 --- a/rerun_py/rerun_sdk/rerun/sinks.py +++ b/rerun_py/rerun_sdk/rerun/sinks.py @@ -261,7 +261,7 @@ def serve( web_port: The port to serve the web viewer on (defaults to 9090). grpc_port: - The port to serve the gRPC server on (defaults to 1852) + The port to serve the gRPC server on (defaults to 9876) default_blueprint: Optionally set a default blueprint to use for this application. If the application already has an active blueprint, the new blueprint won't become active until the user @@ -320,7 +320,7 @@ def serve_web( web_port: The port to serve the web viewer on (defaults to 9090). grpc_port: - The port to serve the gRPC server on (defaults to 1852) + The port to serve the gRPC server on (defaults to 9876) default_blueprint: Optionally set a default blueprint to use for this application. If the application already has an active blueprint, the new blueprint won't become active until the user @@ -462,7 +462,7 @@ def spawn( def spawn_grpc( *, - port: int = 1852, + port: int = 9876, connect: bool = True, memory_limit: str = "75%", hide_welcome_screen: bool = False, From f5671bc4745bfd480c5d610646722b4624152287 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Thu, 30 Jan 2025 19:19:37 +0100 Subject: [PATCH 51/87] Remove WebSocket data source --- crates/store/re_data_source/src/data_source.rs | 11 ----------- crates/store/re_data_source/src/lib.rs | 2 +- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/crates/store/re_data_source/src/data_source.rs b/crates/store/re_data_source/src/data_source.rs index 6bc4c1468cf8..73c93348ce05 100644 --- a/crates/store/re_data_source/src/data_source.rs +++ b/crates/store/re_data_source/src/data_source.rs @@ -275,7 +275,6 @@ fn test_data_source_from_uri() { "www.foo.zip/foo.rrd", "www.foo.zip/blueprint.rbl", ]; - let ws = ["ws://foo.zip", "wss://foo.zip", "127.0.0.1"]; let file_source = FileSource::DragAndDrop { recommended_application_id: None, @@ -302,14 +301,4 @@ fn test_data_source_from_uri() { "Expected {uri:?} to be categorized as RrdHttpUrl" ); } - - for uri in ws { - assert!( - matches!( - DataSource::from_uri(file_source.clone(), uri.to_owned()), - DataSource::WebSocketAddr(_) - ), - "Expected {uri:?} to be categorized as WebSocketAddr" - ); - } } diff --git a/crates/store/re_data_source/src/lib.rs b/crates/store/re_data_source/src/lib.rs index bec3bb0c190a..528a6a0a52cf 100644 --- a/crates/store/re_data_source/src/lib.rs +++ b/crates/store/re_data_source/src/lib.rs @@ -1,7 +1,7 @@ //! Handles different ways of loading Rerun data, e.g.: //! //! - Over HTTPS -//! - Over WebSockets +//! - Over gRPC //! - From disk //! //! Also handles different file types: rrd, images, text files, 3D models, point clouds… From c0ae6d17e3fa66fafa6b1a4bf6391e21b95b2b14 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Thu, 30 Jan 2025 22:56:35 +0100 Subject: [PATCH 52/87] Python: Bring back `connect` and `connect_tcp` as deprecated --- rerun_py/rerun_sdk/rerun/blueprint/api.py | 5 ++ rerun_py/rerun_sdk/rerun/sinks.py | 62 ++++++++++++++++++++--- 2 files changed, 61 insertions(+), 6 deletions(-) diff --git a/rerun_py/rerun_sdk/rerun/blueprint/api.py b/rerun_py/rerun_sdk/rerun/blueprint/api.py index 30a8b2c2243f..8f69a161da1c 100644 --- a/rerun_py/rerun_sdk/rerun/blueprint/api.py +++ b/rerun_py/rerun_sdk/rerun/blueprint/api.py @@ -2,6 +2,7 @@ import uuid from typing import Iterable, Optional, Union +from typing_extensions import deprecated # type: ignore[misc, unused-ignore] import rerun_bindings as bindings @@ -559,6 +560,10 @@ def _ipython_display_(self) -> None: Viewer(blueprint=self).display() + @deprecated( + """Please migrate to `connect_grpc(…)`. + See: https://www.rerun.io/docs/reference/migration/migration-0-22?speculative-link for more details.""" + ) def connect( self, application_id: str, diff --git a/rerun_py/rerun_sdk/rerun/sinks.py b/rerun_py/rerun_sdk/rerun/sinks.py index c13cd884a15a..6e62f34bc64d 100644 --- a/rerun_py/rerun_sdk/rerun/sinks.py +++ b/rerun_py/rerun_sdk/rerun/sinks.py @@ -21,22 +21,29 @@ def is_recording_enabled(recording: RecordingStream | None) -> bool: return bindings.is_enabled() # type: ignore[no-any-return] +@deprecated( + """Please migrate to `connect_grpc(…)`. + See: https://www.rerun.io/docs/reference/migration/migration-0-22?speculative-link for more details.""" +) def connect( - url: str | None = None, + addr: str | None = None, *, + flush_timeout_sec: float | None = 2.0, default_blueprint: BlueprintLike | None = None, recording: RecordingStream | None = None, ) -> None: """ - Connect to a remote Rerun Viewer on the given HTTP(S) URL. + Connect to a remote Rerun Viewer on the given ip:port. This function returns immediately. Parameters ---------- - url: - The HTTP(S) URL to connect to - default_blueprint + addr: + The ip:port to connect to + flush_timeout_sec: + Deprecated. + default_blueprint: Optionally set a default blueprint to use for this application. If the application already has an active blueprint, the new blueprint won't become active until the user clicks the "reset blueprint" button. If you want to activate the new blueprint @@ -47,13 +54,56 @@ def connect( See also: [`rerun.init`][], [`rerun.set_global_data_recording`][]. """ + if addr is not None and not addr.startswith('http'): + addr = f"http://{addr}" return connect_grpc( - url, + url=addr, default_blueprint=default_blueprint, recording=recording, # NOLINT: conversion not needed ) +@deprecated( + """Please migrate to `connect_grpc(…)`. + See: https://www.rerun.io/docs/reference/migration/migration-0-22?speculative-link for more details.""" +) +def connect_tcp( + addr: str | None = None, + *, + flush_timeout_sec: float | None = 2.0, + default_blueprint: BlueprintLike | None = None, + recording: RecordingStream | None = None, +) -> None: + """ + Connect to a remote Rerun Viewer on the given ip:port. + + This function returns immediately. + + Parameters + ---------- + addr: + The ip:port to connect to + flush_timeout_sec: + Deprecated. + default_blueprint: + Optionally set a default blueprint to use for this application. If the application + already has an active blueprint, the new blueprint won't become active until the user + clicks the "reset blueprint" button. If you want to activate the new blueprint + immediately, instead use the [`rerun.send_blueprint`][] API. + recording: + Specifies the [`rerun.RecordingStream`][] to use. + If left unspecified, defaults to the current active data recording, if there is one. + See also: [`rerun.init`][], [`rerun.set_global_data_recording`][]. + + """ + if addr is not None and not addr.startswith('http'): + addr = f"http://{addr}" + return connect_grpc( + url=addr, + default_blueprint=default_blueprint, + recording=recording, # NOLINT: conversion not needed + ) + def connect_grpc( url: str | None = None, *, From 22b966f40661055fc8fd93f6f23033cb5a3b7ea4 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Thu, 30 Jan 2025 23:03:40 +0100 Subject: [PATCH 53/87] Rust: Deprecate `connect_tcp` --- crates/top/re_sdk/src/recording_stream.rs | 61 ++++++++++++++++--- crates/top/rerun/src/lib.rs | 2 +- .../quick_start_guides/quick_start_connect.rs | 3 +- docs/content/getting-started/data-in/rust.md | 4 +- .../all/concepts/app-model/native-sync.rs | 6 +- .../all/quick_start/quick_start_connect.rs | 5 +- .../python/multiprocess_logging/README.md | 2 +- examples/rust/chess_robby_fischer/README.md | 4 +- 8 files changed, 68 insertions(+), 19 deletions(-) diff --git a/crates/top/re_sdk/src/recording_stream.rs b/crates/top/re_sdk/src/recording_stream.rs index dfed098132fc..3fc933aacab3 100644 --- a/crates/top/re_sdk/src/recording_stream.rs +++ b/crates/top/re_sdk/src/recording_stream.rs @@ -304,9 +304,7 @@ impl RecordingStreamBuilder { /// Creates a new [`RecordingStream`] that is pre-configured to stream the data through to a /// remote Rerun instance. /// - /// See also [`Self::connect_opts`] if you wish to configure the connection. - /// - /// This is an alias for [`Self::connect_grpc`]. + /// See also [`Self::connect_opts`] if you wish to configure the TCP connection. /// /// ## Example /// @@ -314,6 +312,7 @@ impl RecordingStreamBuilder { /// let rec = re_sdk::RecordingStreamBuilder::new("rerun_example_app").connect()?; /// # Ok::<(), Box>(()) /// ``` + #[deprecated(since = "0.20.0", note = "use connect_grpc() instead")] pub fn connect(self) -> RecordingStreamResult { self.connect_grpc() } @@ -321,17 +320,65 @@ impl RecordingStreamBuilder { /// Creates a new [`RecordingStream`] that is pre-configured to stream the data through to a /// remote Rerun instance. /// - /// This is an alias for [`Self::connect_grpc_opts`]. + /// See also [`Self::connect_opts`] if you wish to configure the TCP connection. + /// + /// ## Example + /// + /// ```no_run + /// let rec = re_sdk::RecordingStreamBuilder::new("rerun_example_app").connect_tcp()?; + /// # Ok::<(), Box>(()) + /// ``` + #[deprecated(since = "0.22.0", note = "use connect_grpc() instead")] + pub fn connect_tcp(self) -> RecordingStreamResult { + self.connect_grpc() + } + + /// Creates a new [`RecordingStream`] that is pre-configured to stream the data through to a + /// remote Rerun instance. + /// + /// `flush_timeout` is the minimum time the [`TcpSink`][`crate::log_sink::TcpSink`] will + /// wait during a flush before potentially dropping data. Note: Passing `None` here can cause a + /// call to `flush` to block indefinitely if a connection cannot be established. /// /// ## Example /// /// ```no_run /// let rec = re_sdk::RecordingStreamBuilder::new("rerun_example_app") - /// .connect_opts("http://127.0.0.1:9876")?; + /// .connect_opts(re_sdk::default_server_addr(), re_sdk::default_flush_timeout())?; /// # Ok::<(), Box>(()) /// ``` - pub fn connect_opts(self, url: impl Into) -> RecordingStreamResult { - self.connect_grpc_opts(url) + #[deprecated(since = "0.20.0", note = "use connect_tcp_opts() instead")] + pub fn connect_opts( + self, + addr: std::net::SocketAddr, + flush_timeout: Option, + ) -> RecordingStreamResult { + let _ = flush_timeout; + self.connect_grpc_opts(format!("http://{addr}")) + } + + /// Creates a new [`RecordingStream`] that is pre-configured to stream the data through to a + /// remote Rerun instance. + /// + /// `flush_timeout` is the minimum time the [`TcpSink`][`crate::log_sink::TcpSink`] will + /// wait during a flush before potentially dropping data. Note: Passing `None` here can cause a + /// call to `flush` to block indefinitely if a connection cannot be established. + /// + /// ## Example + /// + /// ```no_run + /// let rec = re_sdk::RecordingStreamBuilder::new("rerun_example_app") + /// .connect_opts(re_sdk::default_server_addr(), re_sdk::default_flush_timeout())?; + /// # Ok::<(), Box>(()) + /// ``` + #[deprecated(since = "0.22.0", note = "use connect_grpc() instead")] + pub fn connect_tcp_opts( + self, + addr: std::net::SocketAddr, + flush_timeout: Option, + ) -> RecordingStreamResult { + let _ = flush_timeout; + self.connect_grpc_opts(format!("http://{addr}")) } /// Creates a new [`RecordingStream`] that is pre-configured to stream the data through to a diff --git a/crates/top/rerun/src/lib.rs b/crates/top/rerun/src/lib.rs index 9b299c1e0ed6..8396ba11b986 100644 --- a/crates/top/rerun/src/lib.rs +++ b/crates/top/rerun/src/lib.rs @@ -55,7 +55,7 @@ //! # fn positions() -> Vec { Default::default() } //! # fn colors() -> Vec { Default::default() } //! // Stream log data to an awaiting `rerun` process. -//! let rec = rerun::RecordingStreamBuilder::new("rerun_example_app").connect()?; +//! let rec = rerun::RecordingStreamBuilder::new("rerun_example_app").connect_grpc()?; //! //! let points: Vec = positions(); //! let colors: Vec = colors(); diff --git a/crates/viewer/re_viewer/data/quick_start_guides/quick_start_connect.rs b/crates/viewer/re_viewer/data/quick_start_guides/quick_start_connect.rs index be638bda8c76..b4e4b2c69c27 100644 --- a/crates/viewer/re_viewer/data/quick_start_guides/quick_start_connect.rs +++ b/crates/viewer/re_viewer/data/quick_start_guides/quick_start_connect.rs @@ -4,7 +4,8 @@ use rerun::{demo_util::grid, external::glam}; fn main() -> Result<(), Box> { // Create a new `RecordingStream` which sends data over gRPC to the viewer process. - let rec = rerun::RecordingStreamBuilder::new("rerun_example_quick_start_connect").connect()?; + let rec = + rerun::RecordingStreamBuilder::new("rerun_example_quick_start_connect").connect_grpc()?; // Create some data using the `grid` utility function. let points = grid(glam::Vec3::splat(-10.0), glam::Vec3::splat(10.0), 10); diff --git a/docs/content/getting-started/data-in/rust.md b/docs/content/getting-started/data-in/rust.md index 014452604483..9d4b09cf4c01 100644 --- a/docs/content/getting-started/data-in/rust.md +++ b/docs/content/getting-started/data-in/rust.md @@ -61,12 +61,12 @@ Checkout `rerun --help` for more options. To get going we want to create a [`RecordingStream`](https://docs.rs/rerun/latest/rerun/struct.RecordingStream.html): We can do all of this with the [`rerun::RecordingStreamBuilder::new`](https://docs.rs/rerun/latest/rerun/struct.RecordingStreamBuilder.html#method.new) function which allows us to name the dataset we're working on by setting its [`ApplicationId`](https://docs.rs/rerun/latest/rerun/struct.ApplicationId.html). -We then connect it to the already running Viewer via [`connect_tcp`](https://docs.rs/rerun/latest/rerun/struct.RecordingStreamBuilder.html#method.connect_tcp), returning the `RecordingStream` upon success. +We then connect it to the already running Viewer via [`connect_grpc`](https://docs.rs/rerun/latest/rerun/struct.RecordingStreamBuilder.html#method.connect_grpc?speculative-link), returning the `RecordingStream` upon success. ```rust fn main() -> Result<(), Box> { let rec = rerun::RecordingStreamBuilder::new("rerun_example_dna_abacus") - .connect_tcp()?; + .connect_grpc()?; Ok(()) } diff --git a/docs/snippets/all/concepts/app-model/native-sync.rs b/docs/snippets/all/concepts/app-model/native-sync.rs index 2db8313dd9df..17a2cbe2a996 100644 --- a/docs/snippets/all/concepts/app-model/native-sync.rs +++ b/docs/snippets/all/concepts/app-model/native-sync.rs @@ -1,9 +1,9 @@ fn main() -> Result<(), Box> { - // Connect to the Rerun TCP server using the default address and + // Connect to the Rerun gRPC server using the default address and // port: localhost:9876 - let rec = rerun::RecordingStreamBuilder::new("rerun_example_native_sync").connect()?; + let rec = rerun::RecordingStreamBuilder::new("rerun_example_native_sync").connect_grpc()?; - // Log data as usual, thereby pushing it into the TCP socket. + // Log data as usual, thereby pushing it into the stream. loop { rec.log("/", &rerun::TextLog::new("Logging things..."))?; } diff --git a/docs/snippets/all/quick_start/quick_start_connect.rs b/docs/snippets/all/quick_start/quick_start_connect.rs index 0ccd3064e122..b4e4b2c69c27 100644 --- a/docs/snippets/all/quick_start/quick_start_connect.rs +++ b/docs/snippets/all/quick_start/quick_start_connect.rs @@ -3,8 +3,9 @@ use rerun::{demo_util::grid, external::glam}; fn main() -> Result<(), Box> { - // Create a new `RecordingStream` which sends data over TCP to the viewer process. - let rec = rerun::RecordingStreamBuilder::new("rerun_example_quick_start_connect").connect()?; + // Create a new `RecordingStream` which sends data over gRPC to the viewer process. + let rec = + rerun::RecordingStreamBuilder::new("rerun_example_quick_start_connect").connect_grpc()?; // Create some data using the `grid` utility function. let points = grid(glam::Vec3::splat(-10.0), glam::Vec3::splat(10.0), 10); diff --git a/examples/python/multiprocess_logging/README.md b/examples/python/multiprocess_logging/README.md index e49cbfdcc826..24a217dbeeac 100644 --- a/examples/python/multiprocess_logging/README.md +++ b/examples/python/multiprocess_logging/README.md @@ -29,7 +29,7 @@ The function `task` is decorated with `@rr.shutdown_at_exit`. This decorator ens def task(child_index: int) -> None: rr.init("rerun_example_multiprocessing") - rr.connect_tcp() + rr.connect_grpc() title = f"task_{child_index}" rr.log( diff --git a/examples/rust/chess_robby_fischer/README.md b/examples/rust/chess_robby_fischer/README.md index cdefec85040c..4c94570e88d7 100644 --- a/examples/rust/chess_robby_fischer/README.md +++ b/examples/rust/chess_robby_fischer/README.md @@ -30,7 +30,7 @@ let app_id = "RobbyFischer"; let rec_id = uuid::Uuid::new_v4().to_string(); let rec = rerun::RecordingStreamBuilder::new(app_id) .recording_id(&rec_id) - .connect_tcp() + .connect_grpc() .unwrap(); // … @@ -323,7 +323,7 @@ parser.add_argument("--application-id", type=str) args = parser.parse_args() rr.init(args.application_id, recording_id=args.recording_id) -rr.connect_tcp() +rr.connect_grpc() rr.send_blueprint(blueprint) ``` From fd76dbac285b4b91de528d9fb727a54abbee2a9e Mon Sep 17 00:00:00 2001 From: jprochazk Date: Thu, 30 Jan 2025 23:25:29 +0100 Subject: [PATCH 54/87] C/C++: Bring back `connect`/`connect_tcp` as deprecated --- crates/top/rerun_c/src/lib.rs | 61 +++++++++++++++--------- rerun_cpp/src/rerun/c/rerun.h | 24 +++++++++- rerun_cpp/src/rerun/recording_stream.cpp | 21 +++++--- rerun_cpp/src/rerun/recording_stream.hpp | 36 +++++++++----- 4 files changed, 97 insertions(+), 45 deletions(-) diff --git a/crates/top/rerun_c/src/lib.rs b/crates/top/rerun_c/src/lib.rs index 3e3a1680d17d..fbac99e80ccb 100644 --- a/crates/top/rerun_c/src/lib.rs +++ b/crates/top/rerun_c/src/lib.rs @@ -558,18 +558,27 @@ pub extern "C" fn rr_recording_stream_flush_blocking(id: CRecordingStream) { } #[allow(clippy::result_large_err)] -fn rr_recording_stream_connect_grpc_impl( +fn rr_recording_stream_connect_impl( stream: CRecordingStream, - url: CStringView, + tcp_addr: CStringView, + flush_timeout_sec: f32, ) -> Result<(), CError> { let stream = recording_stream(stream)?; - let url = url.as_str("url")?.parse(); + let tcp_addr = tcp_addr.as_str("tcp_addr")?; + let tcp_addr = tcp_addr.parse().map_err(|err| { + CError::new( + CErrorCode::InvalidSocketAddress, + &format!("Failed to parse tcp address {tcp_addr:?}: {err}"), + ) + })?; - match url { - Ok(url) => stream.connect_grpc_opts(url), - Err(err) => return Err(CError::new(CErrorCode::InvalidServerUrl, &err.to_string())), - } + let flush_timeout = if flush_timeout_sec >= 0.0 { + Some(std::time::Duration::from_secs_f32(flush_timeout_sec)) + } else { + None + }; + stream.connect_opts(tcp_addr, flush_timeout); Ok(()) } @@ -578,14 +587,32 @@ fn rr_recording_stream_connect_grpc_impl( #[no_mangle] pub extern "C" fn rr_recording_stream_connect( id: CRecordingStream, - url: CStringView, + tcp_addr: CStringView, + flush_timeout_sec: f32, error: *mut CError, ) { - if let Err(err) = rr_recording_stream_connect_grpc_impl(id, url) { + if let Err(err) = rr_recording_stream_connect_impl(id, tcp_addr, flush_timeout_sec) { err.write_error(error); } } +#[allow(clippy::result_large_err)] +fn rr_recording_stream_connect_grpc_impl( + stream: CRecordingStream, + url: CStringView, +) -> Result<(), CError> { + let stream = recording_stream(stream)?; + + let url = url.as_str("url")?.parse(); + + match url { + Ok(url) => stream.connect_grpc_opts(url), + Err(err) => return Err(CError::new(CErrorCode::InvalidServerUrl, &err.to_string())), + } + + Ok(()) +} + #[allow(unsafe_code)] #[no_mangle] pub extern "C" fn rr_recording_stream_connect_grpc( @@ -599,7 +626,7 @@ pub extern "C" fn rr_recording_stream_connect_grpc( } #[allow(clippy::result_large_err)] -fn rr_recording_stream_spawn_grpc_impl( +fn rr_recording_stream_spawn_impl( stream: CRecordingStream, spawn_opts: *const CSpawnOptions, ) -> Result<(), CError> { @@ -626,19 +653,7 @@ pub extern "C" fn rr_recording_stream_spawn( spawn_opts: *const CSpawnOptions, error: *mut CError, ) { - if let Err(err) = rr_recording_stream_spawn_grpc_impl(id, spawn_opts) { - err.write_error(error); - } -} - -#[allow(unsafe_code)] -#[no_mangle] -pub extern "C" fn rr_recording_stream_spawn_grpc( - id: CRecordingStream, - spawn_opts: *const CSpawnOptions, - error: *mut CError, -) { - if let Err(err) = rr_recording_stream_spawn_grpc_impl(id, spawn_opts) { + if let Err(err) = rr_recording_stream_spawn_impl(id, spawn_opts) { err.write_error(error); } } diff --git a/rerun_cpp/src/rerun/c/rerun.h b/rerun_cpp/src/rerun/c/rerun.h index 95884b67f6ae..ee12ee3eecea 100644 --- a/rerun_cpp/src/rerun/c/rerun.h +++ b/rerun_cpp/src/rerun/c/rerun.h @@ -412,6 +412,21 @@ extern void rr_recording_stream_set_thread_local( /// Check whether the recording stream is enabled. extern bool rr_recording_stream_is_enabled(rr_recording_stream stream, rr_error* error); +/// Connect to a remote Rerun Viewer on the given ip:port. +/// +/// Requires that you first start a Rerun Viewer by typing 'rerun' in a terminal. +/// +/// flush_timeout_sec: +/// The minimum time the SDK will wait during a flush before potentially +/// dropping data if progress is not being made. Passing a negative value indicates no timeout, +/// and can cause a call to `flush` to block indefinitely. +/// +/// This function returns immediately and will only raise an error for argument parsing errors, +/// not for connection errors as these happen asynchronously. +extern void rr_recording_stream_connect( + rr_recording_stream stream, rr_string tcp_addr, float flush_timeout_sec, rr_error* error +) __attribute__((deprecated)); + /// Connect to a remote Rerun Viewer on the given HTTP(S) URL. /// /// Requires that you first start a Rerun Viewer by typing 'rerun' in a terminal. @@ -428,7 +443,7 @@ extern void rr_recording_stream_connect_grpc( ); /// Spawns a new Rerun Viewer process from an executable available in PATH, then connects to it -/// over gRPC. +/// over TCP. /// /// This function returns immediately and will only raise an error for argument parsing errors, /// not for connection errors as these happen asynchronously. @@ -439,7 +454,12 @@ extern void rr_recording_stream_connect_grpc( /// Configuration of the spawned process. /// Refer to `rr_spawn_options` documentation for details. /// Passing null is valid and will result in the recommended defaults. -extern void rr_recording_stream_spawn_grpc( +/// +/// flush_timeout_sec: +/// The minimum time the SDK will wait during a flush before potentially +/// dropping data if progress is not being made. Passing a negative value indicates no timeout, +/// and can cause a call to `flush` to block indefinitely. +extern void rr_recording_stream_spawn( rr_recording_stream stream, const rr_spawn_options* spawn_opts, rr_error* error ); diff --git a/rerun_cpp/src/rerun/recording_stream.cpp b/rerun_cpp/src/rerun/recording_stream.cpp index 08355863928b..2846f08bd5c6 100644 --- a/rerun_cpp/src/rerun/recording_stream.cpp +++ b/rerun_cpp/src/rerun/recording_stream.cpp @@ -104,8 +104,19 @@ namespace rerun { } } - Error RecordingStream::connect(std::string_view url) const { - return connect_grpc(url); + Error RecordingStream::connect(std::string_view tcp_addr, float flush_timeout_sec) const { + return RecordingStream::connect_tcp(tcp_addr, flush_timeout_sec); + } + + Error RecordingStream::connect_tcp(std::string_view tcp_addr, float flush_timeout_sec) const { + rr_error status = {}; + rr_recording_stream_connect( + _id, + detail::to_rr_string(tcp_addr), + flush_timeout_sec, + &status + ); + return status; } Error RecordingStream::connect_grpc(std::string_view url) const { @@ -115,14 +126,10 @@ namespace rerun { } Error RecordingStream::spawn(const SpawnOptions& options) const { - return spawn_grpc(options); - } - - Error RecordingStream::spawn_grpc(const SpawnOptions& options) const { rr_spawn_options rerun_c_options = {}; options.fill_rerun_c_struct(rerun_c_options); rr_error status = {}; - rr_recording_stream_spawn_grpc(_id, &rerun_c_options, &status); + rr_recording_stream_spawn(_id, &rerun_c_options, &status); return status; } diff --git a/rerun_cpp/src/rerun/recording_stream.hpp b/rerun_cpp/src/rerun/recording_stream.hpp index ca0582f49c8e..29d16d3066dc 100644 --- a/rerun_cpp/src/rerun/recording_stream.hpp +++ b/rerun_cpp/src/rerun/recording_stream.hpp @@ -133,30 +133,40 @@ namespace rerun { /// \details Either of these needs to be called, otherwise the stream will buffer up indefinitely. /// @{ - /// Connect to a remote Rerun Viewer on the given HTTP(S) URL. + /// Connect to a remote Rerun Viewer on the given ip:port. /// /// Requires that you first start a Rerun Viewer by typing 'rerun' in a terminal. /// + /// flush_timeout_sec: + /// The minimum time the SDK will wait during a flush before potentially + /// dropping data if progress is not being made. Passing a negative value indicates no + /// timeout, and can cause a call to `flush` to block indefinitely. + /// /// This function returns immediately. - Error connect(std::string_view url = "http://127.0.0.1:9876") const; + [[deprecated("Use `connect_grpc` instead")]] Error connect( + std::string_view tcp_addr = "127.0.0.1:9876", float flush_timeout_sec = 2.0 + ) const; - /// Connect to a remote Rerun Viewer on the given HTTP(S) URL. + /// Connect to a remote Rerun Viewer on the given ip:port. /// /// Requires that you first start a Rerun Viewer by typing 'rerun' in a terminal. /// + /// flush_timeout_sec: + /// The minimum time the SDK will wait during a flush before potentially + /// dropping data if progress is not being made. Passing a negative value indicates no + /// timeout, and can cause a call to `flush` to block indefinitely. + /// /// This function returns immediately. - Error connect_grpc(std::string_view url = "http://127.0.0.1:9876") const; + [[deprecated("Use `connect_grpc` instead")]] Error connect_tcp( + std::string_view tcp_addr = "127.0.0.1:9876", float flush_timeout_sec = 2.0 + ) const; - /// Spawns a new Rerun Viewer process from an executable available in PATH, then connects to it - /// over gRPC. + /// Connect to a remote Rerun Viewer on the given HTTP(S) URL. /// - /// If a Rerun Viewer is already listening on this port, the stream will be redirected to - /// that viewer instead of starting a new one. + /// Requires that you first start a Rerun Viewer by typing 'rerun' in a terminal. /// - /// ## Parameters - /// options: - /// See `rerun::SpawnOptions` for more information. - Error spawn(const SpawnOptions& options = {}) const; + /// This function returns immediately. + Error connect_grpc(std::string_view url = "http://127.0.0.1:9876") const; /// Spawns a new Rerun Viewer process from an executable available in PATH, then connects to it /// over gRPC. @@ -167,7 +177,7 @@ namespace rerun { /// ## Parameters /// options: /// See `rerun::SpawnOptions` for more information. - Error spawn_grpc(const SpawnOptions& options = {}) const; + Error spawn(const SpawnOptions& options = {}) const; /// @see RecordingStream::spawn template From 236ba06241a3242063a6c15b371783df2fe0843d Mon Sep 17 00:00:00 2001 From: jprochazk Date: Thu, 30 Jan 2025 23:26:47 +0100 Subject: [PATCH 55/87] Rust: Bring back `connect` as deprecated on `RecordingStream` --- crates/top/re_sdk/src/recording_stream.rs | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/crates/top/re_sdk/src/recording_stream.rs b/crates/top/re_sdk/src/recording_stream.rs index 3fc933aacab3..477c8be15c95 100644 --- a/crates/top/re_sdk/src/recording_stream.rs +++ b/crates/top/re_sdk/src/recording_stream.rs @@ -1714,26 +1714,37 @@ impl RecordingStream { } impl RecordingStream { - /// Swaps the underlying sink for a [`crate::log_sink::GrpcSink`] sink pre-configured to use + /// Swaps the underlying sink for a [`crate::log_sink::TcpSink`] sink pre-configured to use /// the specified address. /// - /// See also [`Self::connect_opts`] if you wish to configure the connection. + /// See also [`Self::connect_opts`] if you wish to configure the TCP connection. /// /// This is a convenience wrapper for [`Self::set_sink`] that upholds the same guarantees in /// terms of data durability and ordering. /// See [`Self::set_sink`] for more information. + #[deprecated(since = "0.22.0", note = "use connect_grpc() instead")] pub fn connect(&self) { self.connect_grpc(); } - /// Swaps the underlying sink for a [`crate::log_sink::GrpcSink`] sink pre-configured to use + /// Swaps the underlying sink for a [`crate::log_sink::TcpSink`] sink pre-configured to use /// the specified address. /// + /// `flush_timeout` is the minimum time the [`TcpSink`][`crate::log_sink::TcpSink`] will + /// wait during a flush before potentially dropping data. Note: Passing `None` here can cause a + /// call to `flush` to block indefinitely if a connection cannot be established. + /// /// This is a convenience wrapper for [`Self::set_sink`] that upholds the same guarantees in /// terms of data durability and ordering. /// See [`Self::set_sink`] for more information. - pub fn connect_opts(&self, url: re_grpc_client::MessageProxyUrl) { - self.connect_grpc_opts(url); + #[deprecated(since = "0.22.0", note = "use connect_grpc() instead")] + pub fn connect_opts( + &self, + addr: std::net::SocketAddr, + flush_timeout: Option, + ) { + let _ = flush_timeout; + self.connect_grpc_opts(format!("http://{addr}")); } /// Swaps the underlying sink for a [`crate::log_sink::GrpcSink`] sink pre-configured to use From 266124ab5afd947a72fe4b014ed955c34167b43b Mon Sep 17 00:00:00 2001 From: jprochazk Date: Thu, 30 Jan 2025 23:27:25 +0100 Subject: [PATCH 56/87] Fix compile error --- crates/top/re_sdk/src/recording_stream.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/top/re_sdk/src/recording_stream.rs b/crates/top/re_sdk/src/recording_stream.rs index 477c8be15c95..5ba3b7153a5b 100644 --- a/crates/top/re_sdk/src/recording_stream.rs +++ b/crates/top/re_sdk/src/recording_stream.rs @@ -1744,7 +1744,11 @@ impl RecordingStream { flush_timeout: Option, ) { let _ = flush_timeout; - self.connect_grpc_opts(format!("http://{addr}")); + self.connect_grpc_opts( + format!("http://{addr}") + .parse() + .expect("should always be valid"), + ); } /// Swaps the underlying sink for a [`crate::log_sink::GrpcSink`] sink pre-configured to use From 4bf32440758c96290c930bb4ad650eca5a4efced Mon Sep 17 00:00:00 2001 From: jprochazk Date: Thu, 30 Jan 2025 23:38:45 +0100 Subject: [PATCH 57/87] C/C++: Fix tests after changes, add new test for grpc connection --- crates/top/rerun_c/src/lib.rs | 4 +- rerun_cpp/docs/readme_snippets.cpp | 4 +- rerun_cpp/src/rerun/error.hpp | 3 +- rerun_cpp/src/rerun/recording_stream.cpp | 1 + rerun_cpp/tests/recording_stream.cpp | 139 ++++++++++++++++------- 5 files changed, 106 insertions(+), 45 deletions(-) diff --git a/crates/top/rerun_c/src/lib.rs b/crates/top/rerun_c/src/lib.rs index fbac99e80ccb..8eae28095885 100644 --- a/crates/top/rerun_c/src/lib.rs +++ b/crates/top/rerun_c/src/lib.rs @@ -281,8 +281,9 @@ pub enum CErrorCode { InvalidStringArgument, InvalidEnumValue, InvalidRecordingStreamHandle, - InvalidServerUrl, + InvalidSocketAddress, InvalidComponentTypeHandle, + InvalidServerUrl = 0x0000_0001a, _CategoryRecordingStream = 0x0000_00100, RecordingStreamRuntimeFailure, @@ -578,6 +579,7 @@ fn rr_recording_stream_connect_impl( } else { None }; + #[expect(deprecated)] // Will be removed once `connect` is removed. stream.connect_opts(tcp_addr, flush_timeout); Ok(()) diff --git a/rerun_cpp/docs/readme_snippets.cpp b/rerun_cpp/docs/readme_snippets.cpp index b4e676dd4431..5d79c9276416 100644 --- a/rerun_cpp/docs/readme_snippets.cpp +++ b/rerun_cpp/docs/readme_snippets.cpp @@ -21,7 +21,7 @@ static std::vector create_image() { // Create a recording stream. rerun::RecordingStream rec("rerun_example_app"); // Spawn the viewer and connect to it. - rec.spawn_grpc().exit_on_failure(); + rec.spawn().exit_on_failure(); std::vector points = create_positions(); std::vector colors = create_colors(); @@ -63,7 +63,7 @@ static std::vector create_image() { rec.log("path/to/points", rerun::Points3D(points).with_colors(colors)); // Spawn & connect later. - auto result = rec.spawn_grpc(); + auto result = rec.spawn(); if (result.is_err()) { // Handle error. } diff --git a/rerun_cpp/src/rerun/error.hpp b/rerun_cpp/src/rerun/error.hpp index 4330d2ae2590..cb9f61de3582 100644 --- a/rerun_cpp/src/rerun/error.hpp +++ b/rerun_cpp/src/rerun/error.hpp @@ -38,11 +38,12 @@ namespace rerun { InvalidStringArgument, InvalidEnumValue, InvalidRecordingStreamHandle, - InvalidServerUrl, + InvalidSocketAddress, InvalidComponentTypeHandle, InvalidTensorDimension, InvalidArchetypeField, FileRead, + InvalidServerUrl, // Recording stream errors _CategoryRecordingStream = 0x0000'0100, diff --git a/rerun_cpp/src/rerun/recording_stream.cpp b/rerun_cpp/src/rerun/recording_stream.cpp index 2846f08bd5c6..184f1d32b52a 100644 --- a/rerun_cpp/src/rerun/recording_stream.cpp +++ b/rerun_cpp/src/rerun/recording_stream.cpp @@ -104,6 +104,7 @@ namespace rerun { } } + RR_DISABLE_DEPRECATION_WARNING Error RecordingStream::connect(std::string_view tcp_addr, float flush_timeout_sec) const { return RecordingStream::connect_tcp(tcp_addr, flush_timeout_sec); } diff --git a/rerun_cpp/tests/recording_stream.cpp b/rerun_cpp/tests/recording_stream.cpp index 3e87bfd84710..5a38b5e85a8a 100644 --- a/rerun_cpp/tests/recording_stream.cpp +++ b/rerun_cpp/tests/recording_stream.cpp @@ -147,13 +147,14 @@ SCENARIO("RecordingStream can be used for logging archetypes and components", TE rerun::RecordingStream stream("test", std::string_view(), kind); GIVEN("component batches") { - auto batch0 = rerun::ComponentBatch::from_loggable( - {{1.0, 2.0}, {4.0, 5.0}} - ).value_or_throw(); - auto batch1 = rerun::ComponentBatch::from_loggable( - {rerun::Color(0xFF0000FF)} - ) - .value_or_throw(); + auto batch0 = + rerun::ComponentBatch::from_loggable({{1.0, 2.0}, + {4.0, 5.0}}) + .value_or_throw(); + auto batch1 = + rerun::ComponentBatch::from_loggable({rerun::Color(0xFF0000FF) + }) + .value_or_throw(); THEN("single component batch can be logged") { stream.log("log_archetype-splat", batch0); stream.log_static("log_archetype-splat", batch0); @@ -172,9 +173,9 @@ SCENARIO("RecordingStream can be used for logging archetypes and components", TE auto batch0 = rerun::ComponentBatch::from_loggable( {{1.0, 2.0}, {4.0, 5.0}} ); - auto batch1 = rerun::ComponentBatch::from_loggable( - {rerun::Color(0xFF0000FF)} - ); + auto batch1 = + rerun::ComponentBatch::from_loggable({rerun::Color(0xFF0000FF) + }); THEN("single component batch can be logged") { stream.log("log_archetype-splat", batch0); stream.log_static("log_archetype-splat", batch0); @@ -186,7 +187,8 @@ SCENARIO("RecordingStream can be used for logging archetypes and components", TE THEN("collection of component batch results can be logged") { rerun::Collection> batches = { batch0, - batch1}; + batch1 + }; stream.log("log_archetype-splat", batches); stream.log_static("log_archetype-splat", batches); } @@ -195,29 +197,29 @@ SCENARIO("RecordingStream can be used for logging archetypes and components", TE THEN("an archetype can be logged") { stream.log( "log_archetype-splat", - rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}} - ).with_colors(rerun::Color(0xFF0000FF)) + rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}}) + .with_colors(rerun::Color(0xFF0000FF)) ); stream.log_static( "log_archetype-splat", - rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}} - ).with_colors(rerun::Color(0xFF0000FF)) + rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}}) + .with_colors(rerun::Color(0xFF0000FF)) ); } THEN("several archetypes can be logged") { stream.log( "log_archetype-splat", - rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}} - ).with_colors(rerun::Color(0xFF0000FF)), - rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}} - ).with_colors(rerun::Color(0xFF0000FF)) + rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}}) + .with_colors(rerun::Color(0xFF0000FF)), + rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}}) + .with_colors(rerun::Color(0xFF0000FF)) ); stream.log_static( "log_archetype-splat", - rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}} - ).with_colors(rerun::Color(0xFF0000FF)), - rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}} - ).with_colors(rerun::Color(0xFF0000FF)) + rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}}) + .with_colors(rerun::Color(0xFF0000FF)), + rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}}) + .with_colors(rerun::Color(0xFF0000FF)) ); } @@ -297,26 +299,81 @@ SCENARIO("RecordingStream can log to file", TEST_TAG) { } } -void test_logging_to_connection(const char* url, const rerun::RecordingStream& stream) { - // We changed to taking std::string_view instead of const char* and constructing such from nullptr crashes - // at least on some C++ implementations. - // If we'd want to support this in earnest we'd have to create out own string_view type. - // - // AND_GIVEN("a nullptr for the socket url") { - // THEN("then the connect call returns a null argument error") { - // CHECK(stream.connect(nullptr, 0.0f).code == rerun::ErrorCode::UnexpectedNullArgument); - // } - // } +void test_logging_to_connection(const char* address, const rerun::RecordingStream& stream) { + RR_DISABLE_DEPRECATION_WARNING // TODO(jan): Remove once `connect` is removed + { + // We changed to taking std::string_view instead of const char* and constructing such from nullptr crashes + // at least on some C++ implementations. + // If we'd want to support this in earnest we'd have to create out own string_view type. + // + // AND_GIVEN("a nullptr for the socket address") { + // THEN("then the connect call returns a null argument error") { + // CHECK(stream.connect(nullptr, 0.0f).code == rerun::ErrorCode::UnexpectedNullArgument); + // } + // } + AND_GIVEN("an invalid address for the socket address") { + THEN("connect call fails") { + CHECK( + stream.connect("definitely not valid!").code == + rerun::ErrorCode::InvalidSocketAddress + ); + } + } + AND_GIVEN("a valid socket address " << address) { + THEN("connect call returns no error") { + CHECK(stream.connect(address).code == rerun::ErrorCode::Ok); + + WHEN("logging an archetype and then flushing") { + check_logged_error([&] { + stream.log( + "archetype", + rerun::Points2D({ + rerun::Vec2D{1.0, 2.0}, + rerun::Vec2D{4.0, 5.0}, + }) + ); + }); + + stream.flush_blocking(); + + THEN("does not crash") { + // No easy way to see if it got sent. + } + } + } + } + } +} + +SCENARIO("RecordingStream can connect", TEST_TAG) { + const char* address = "127.0.0.1:9876"; + GIVEN("a new RecordingStream") { + rerun::RecordingStream stream("test-local"); + test_logging_to_connection(address, stream); + } + WHEN("setting a global RecordingStream and then discarding it") { + { + rerun::RecordingStream stream("test-global"); + stream.set_global(); + } + GIVEN("the current recording stream") { + test_logging_to_connection(address, rerun::RecordingStream::current()); + } + } +} + +void test_logging_to_grpc_connection(const char* url, const rerun::RecordingStream& stream) { AND_GIVEN("an invalid url") { - THEN("then the save call fails") { + THEN("connect call fails") { CHECK( - stream.connect("definitely not valid!").code == rerun::ErrorCode::InvalidServerUrl + stream.connect_grpc("definitely not valid!").code == + rerun::ErrorCode::InvalidServerUrl ); } } - AND_GIVEN("a valid url " << url) { - THEN("save call with zero timeout returns no error") { - CHECK(stream.connect(url).code == rerun::ErrorCode::Ok); + AND_GIVEN("a valid socket url " << url) { + THEN("connect call returns no error") { + CHECK(stream.connect_grpc(url).code == rerun::ErrorCode::Ok); WHEN("logging an archetype and then flushing") { check_logged_error([&] { @@ -339,11 +396,11 @@ void test_logging_to_connection(const char* url, const rerun::RecordingStream& s } } -SCENARIO("RecordingStream can connect", TEST_TAG) { +SCENARIO("RecordingStream can connect over grpc", TEST_TAG) { const char* url = "http://127.0.0.1:9876"; GIVEN("a new RecordingStream") { rerun::RecordingStream stream("test-local"); - test_logging_to_connection(url, stream); + test_logging_to_grpc_connection(url, stream); } WHEN("setting a global RecordingStream and then discarding it") { { @@ -351,7 +408,7 @@ SCENARIO("RecordingStream can connect", TEST_TAG) { stream.set_global(); } GIVEN("the current recording stream") { - test_logging_to_connection(url, rerun::RecordingStream::current()); + test_logging_to_grpc_connection(url, rerun::RecordingStream::current()); } } } From 1950414eea6f9c268a7f67779479ccbd138eec4a Mon Sep 17 00:00:00 2001 From: jprochazk Date: Thu, 30 Jan 2025 23:39:57 +0100 Subject: [PATCH 58/87] Run fmt --- rerun_cpp/tests/recording_stream.cpp | 48 +++++++++++------------ rerun_py/rerun_sdk/rerun/blueprint/api.py | 2 +- rerun_py/rerun_sdk/rerun/sinks.py | 5 ++- 3 files changed, 27 insertions(+), 28 deletions(-) diff --git a/rerun_cpp/tests/recording_stream.cpp b/rerun_cpp/tests/recording_stream.cpp index 5a38b5e85a8a..6f4a1151eec0 100644 --- a/rerun_cpp/tests/recording_stream.cpp +++ b/rerun_cpp/tests/recording_stream.cpp @@ -147,14 +147,13 @@ SCENARIO("RecordingStream can be used for logging archetypes and components", TE rerun::RecordingStream stream("test", std::string_view(), kind); GIVEN("component batches") { - auto batch0 = - rerun::ComponentBatch::from_loggable({{1.0, 2.0}, - {4.0, 5.0}}) - .value_or_throw(); - auto batch1 = - rerun::ComponentBatch::from_loggable({rerun::Color(0xFF0000FF) - }) - .value_or_throw(); + auto batch0 = rerun::ComponentBatch::from_loggable( + {{1.0, 2.0}, {4.0, 5.0}} + ).value_or_throw(); + auto batch1 = rerun::ComponentBatch::from_loggable( + {rerun::Color(0xFF0000FF)} + ) + .value_or_throw(); THEN("single component batch can be logged") { stream.log("log_archetype-splat", batch0); stream.log_static("log_archetype-splat", batch0); @@ -173,9 +172,9 @@ SCENARIO("RecordingStream can be used for logging archetypes and components", TE auto batch0 = rerun::ComponentBatch::from_loggable( {{1.0, 2.0}, {4.0, 5.0}} ); - auto batch1 = - rerun::ComponentBatch::from_loggable({rerun::Color(0xFF0000FF) - }); + auto batch1 = rerun::ComponentBatch::from_loggable( + {rerun::Color(0xFF0000FF)} + ); THEN("single component batch can be logged") { stream.log("log_archetype-splat", batch0); stream.log_static("log_archetype-splat", batch0); @@ -187,8 +186,7 @@ SCENARIO("RecordingStream can be used for logging archetypes and components", TE THEN("collection of component batch results can be logged") { rerun::Collection> batches = { batch0, - batch1 - }; + batch1}; stream.log("log_archetype-splat", batches); stream.log_static("log_archetype-splat", batches); } @@ -197,29 +195,29 @@ SCENARIO("RecordingStream can be used for logging archetypes and components", TE THEN("an archetype can be logged") { stream.log( "log_archetype-splat", - rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}}) - .with_colors(rerun::Color(0xFF0000FF)) + rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}} + ).with_colors(rerun::Color(0xFF0000FF)) ); stream.log_static( "log_archetype-splat", - rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}}) - .with_colors(rerun::Color(0xFF0000FF)) + rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}} + ).with_colors(rerun::Color(0xFF0000FF)) ); } THEN("several archetypes can be logged") { stream.log( "log_archetype-splat", - rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}}) - .with_colors(rerun::Color(0xFF0000FF)), - rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}}) - .with_colors(rerun::Color(0xFF0000FF)) + rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}} + ).with_colors(rerun::Color(0xFF0000FF)), + rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}} + ).with_colors(rerun::Color(0xFF0000FF)) ); stream.log_static( "log_archetype-splat", - rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}}) - .with_colors(rerun::Color(0xFF0000FF)), - rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}}) - .with_colors(rerun::Color(0xFF0000FF)) + rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}} + ).with_colors(rerun::Color(0xFF0000FF)), + rerun::Points2D({rerun::Vec2D{1.0, 2.0}, rerun::Vec2D{4.0, 5.0}} + ).with_colors(rerun::Color(0xFF0000FF)) ); } diff --git a/rerun_py/rerun_sdk/rerun/blueprint/api.py b/rerun_py/rerun_sdk/rerun/blueprint/api.py index 8f69a161da1c..302b45e8435d 100644 --- a/rerun_py/rerun_sdk/rerun/blueprint/api.py +++ b/rerun_py/rerun_sdk/rerun/blueprint/api.py @@ -2,9 +2,9 @@ import uuid from typing import Iterable, Optional, Union -from typing_extensions import deprecated # type: ignore[misc, unused-ignore] import rerun_bindings as bindings +from typing_extensions import deprecated # type: ignore[misc, unused-ignore] from .._baseclasses import AsComponents, ComponentBatchLike from .._spawn import _spawn_viewer diff --git a/rerun_py/rerun_sdk/rerun/sinks.py b/rerun_py/rerun_sdk/rerun/sinks.py index 6e62f34bc64d..7af4695abc02 100644 --- a/rerun_py/rerun_sdk/rerun/sinks.py +++ b/rerun_py/rerun_sdk/rerun/sinks.py @@ -54,7 +54,7 @@ def connect( See also: [`rerun.init`][], [`rerun.set_global_data_recording`][]. """ - if addr is not None and not addr.startswith('http'): + if addr is not None and not addr.startswith("http"): addr = f"http://{addr}" return connect_grpc( url=addr, @@ -96,7 +96,7 @@ def connect_tcp( See also: [`rerun.init`][], [`rerun.set_global_data_recording`][]. """ - if addr is not None and not addr.startswith('http'): + if addr is not None and not addr.startswith("http"): addr = f"http://{addr}" return connect_grpc( url=addr, @@ -104,6 +104,7 @@ def connect_tcp( recording=recording, # NOLINT: conversion not needed ) + def connect_grpc( url: str | None = None, *, From a74bb230275794b3426b312d77df00a4f73a0563 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Thu, 30 Jan 2025 23:48:56 +0100 Subject: [PATCH 59/87] Update data source parser to handle grpc - `.rrd` suffix now required for rrd-over-http - any other `http` url is considered a connection to message proxy --- .../store/re_data_source/src/data_source.rs | 64 ++++++++++++------- 1 file changed, 40 insertions(+), 24 deletions(-) diff --git a/crates/store/re_data_source/src/data_source.rs b/crates/store/re_data_source/src/data_source.rs index 73c93348ce05..985ce46d9024 100644 --- a/crates/store/re_data_source/src/data_source.rs +++ b/crates/store/re_data_source/src/data_source.rs @@ -95,21 +95,17 @@ impl DataSource { return Self::RerunGrpcUrl { url: uri }; } - // TODO(#8761): URL prefix - if uri.starts_with("temp://") { - return Self::MessageProxy { url: uri }; - } - if uri.starts_with("file://") || path.exists() { Self::FilePath(file_source, path) - } else if uri.starts_with("http://") - || uri.starts_with("https://") - || (uri.starts_with("www.") && (uri.ends_with(".rrd") || uri.ends_with(".rbl"))) + } else if (uri.starts_with("http://") || uri.starts_with("https://")) + && (uri.ends_with(".rrd") || uri.ends_with(".rbl")) { Self::RrdHttpUrl { url: uri, follow: false, } + } else if uri.starts_with("http://") || uri.starts_with("https://") { + Self::MessageProxy { url: uri } } else if looks_like_a_file_path(&uri) { Self::FilePath(file_source, path) } else if uri.ends_with(".rrd") || uri.ends_with(".rbl") { @@ -261,6 +257,8 @@ impl DataSource { fn test_data_source_from_uri() { use re_log_types::FileSource; + let mut failed = false; + let file = [ "file://foo", "foo.rrd", @@ -269,12 +267,16 @@ fn test_data_source_from_uri() { "D:/file", ]; let http = [ - "http://foo.zip", - "https://foo.zip", "example.zip/foo.rrd", "www.foo.zip/foo.rrd", "www.foo.zip/blueprint.rbl", ]; + let grpc = [ + "http://foo.zip", + "https://foo.zip", + "http://127.0.0.1:9876", + "https://redap.rerun.io", + ]; let file_source = FileSource::DragAndDrop { recommended_application_id: None, @@ -283,22 +285,36 @@ fn test_data_source_from_uri() { }; for uri in file { - assert!( - matches!( - DataSource::from_uri(file_source.clone(), uri.to_owned()), - DataSource::FilePath { .. } - ), - "Expected {uri:?} to be categorized as FilePath" - ); + if !matches!( + DataSource::from_uri(file_source.clone(), uri.to_owned()), + DataSource::FilePath { .. } + ) { + eprintln!("Expected {uri:?} to be categorized as FilePath"); + failed = true; + } } for uri in http { - assert!( - matches!( - DataSource::from_uri(file_source.clone(), uri.to_owned()), - DataSource::RrdHttpUrl { .. } - ), - "Expected {uri:?} to be categorized as RrdHttpUrl" - ); + if !matches!( + DataSource::from_uri(file_source.clone(), uri.to_owned()), + DataSource::RrdHttpUrl { .. } + ) { + eprintln!("Expected {uri:?} to be categorized as RrdHttpUrl"); + failed = true; + } + } + + for uri in grpc { + if !matches!( + DataSource::from_uri(file_source.clone(), uri.to_owned()), + DataSource::MessageProxy { .. } + ) { + eprintln!("Expected {uri:?} to be categorized as MessageProxy"); + failed = true; + } + } + + if failed { + panic!("one or more test cases failed"); } } From 3d4e245b7e23f6b2e7ec893f501fa1dcb81aba0e Mon Sep 17 00:00:00 2001 From: jprochazk Date: Thu, 30 Jan 2025 23:50:47 +0100 Subject: [PATCH 60/87] Remove `spawn_grpc` in favor of just `spawn` --- crates/top/re_sdk/src/recording_stream.rs | 38 +---------------- crates/top/rerun_c/src/lib.rs | 2 +- rerun_py/rerun_sdk/rerun/__init__.py | 2 +- rerun_py/rerun_sdk/rerun/blueprint/api.py | 24 ----------- rerun_py/rerun_sdk/rerun/sinks.py | 51 ----------------------- 5 files changed, 4 insertions(+), 113 deletions(-) diff --git a/crates/top/re_sdk/src/recording_stream.rs b/crates/top/re_sdk/src/recording_stream.rs index 5ba3b7153a5b..366da29d6d0c 100644 --- a/crates/top/re_sdk/src/recording_stream.rs +++ b/crates/top/re_sdk/src/recording_stream.rs @@ -1791,14 +1791,14 @@ impl RecordingStream { /// If a Rerun Viewer is already listening on this port, the stream will be redirected to /// that viewer instead of starting a new one. /// - /// See also [`Self::spawn_grpc_opts`] if you wish to configure the behavior of thew Rerun process + /// See also [`Self::spawn_opts`] if you wish to configure the behavior of thew Rerun process /// as well as the underlying connection. /// /// This is a convenience wrapper for [`Self::set_sink`] that upholds the same guarantees in /// terms of data durability and ordering. /// See [`Self::set_sink`] for more information. pub fn spawn(&self) -> RecordingStreamResult<()> { - self.spawn_grpc() + self.spawn_opts(&Default::default()) } /// Spawns a new Rerun Viewer process from an executable available in PATH, then swaps the @@ -1815,40 +1815,6 @@ impl RecordingStream { /// terms of data durability and ordering. /// See [`Self::set_sink`] for more information. pub fn spawn_opts(&self, opts: &crate::SpawnOptions) -> RecordingStreamResult<()> { - self.spawn_grpc_opts(opts) - } - - /// Spawns a new Rerun Viewer process from an executable available in PATH, then swaps the - /// underlying sink for a [`crate::log_sink::GrpcSink`] sink pre-configured to send data to that - /// new process. - /// - /// If a Rerun Viewer is already listening on this port, the stream will be redirected to - /// that viewer instead of starting a new one. - /// - /// See also [`Self::spawn_grpc_opts`] if you wish to configure the behavior of thew Rerun process - /// as well as the underlying connection. - /// - /// This is a convenience wrapper for [`Self::set_sink`] that upholds the same guarantees in - /// terms of data durability and ordering. - /// See [`Self::set_sink`] for more information. - pub fn spawn_grpc(&self) -> RecordingStreamResult<()> { - self.spawn_grpc_opts(&Default::default()) - } - - /// Spawns a new Rerun Viewer process from an executable available in PATH, then swaps the - /// underlying sink for a [`crate::log_sink::GrpcSink`] sink pre-configured to send data to that - /// new process. - /// - /// If a Rerun Viewer is already listening on this port, the stream will be redirected to - /// that viewer instead of starting a new one. - /// - /// The behavior of the spawned Viewer can be configured via `opts`. - /// If you're fine with the default behavior, refer to the simpler [`Self::spawn`]. - /// - /// This is a convenience wrapper for [`Self::set_sink`] that upholds the same guarantees in - /// terms of data durability and ordering. - /// See [`Self::set_sink`] for more information. - pub fn spawn_grpc_opts(&self, opts: &crate::SpawnOptions) -> RecordingStreamResult<()> { if !self.is_enabled() { re_log::debug!("Rerun disabled - call to spawn() ignored"); return Ok(()); diff --git a/crates/top/rerun_c/src/lib.rs b/crates/top/rerun_c/src/lib.rs index 8eae28095885..2ea4ffa754ae 100644 --- a/crates/top/rerun_c/src/lib.rs +++ b/crates/top/rerun_c/src/lib.rs @@ -642,7 +642,7 @@ fn rr_recording_stream_spawn_impl( }; stream - .spawn_grpc_opts(&spawn_opts) + .spawn_opts(&spawn_opts) .map_err(|err| CError::new(CErrorCode::RecordingStreamSpawnFailure, &err.to_string()))?; Ok(()) diff --git a/rerun_py/rerun_sdk/rerun/__init__.py b/rerun_py/rerun_sdk/rerun/__init__.py index d4ea3e1c3d0e..f2abca72cda0 100644 --- a/rerun_py/rerun_sdk/rerun/__init__.py +++ b/rerun_py/rerun_sdk/rerun/__init__.py @@ -352,7 +352,7 @@ def init( ) if spawn: - from rerun.sinks import spawn_grpc as _spawn + from rerun.sinks import spawn as _spawn _spawn(default_blueprint=default_blueprint) diff --git a/rerun_py/rerun_sdk/rerun/blueprint/api.py b/rerun_py/rerun_sdk/rerun/blueprint/api.py index 302b45e8435d..23f40495415b 100644 --- a/rerun_py/rerun_sdk/rerun/blueprint/api.py +++ b/rerun_py/rerun_sdk/rerun/blueprint/api.py @@ -675,30 +675,6 @@ def spawn( """ Spawn a Rerun viewer with this blueprint. - Parameters - ---------- - application_id: - The application ID to use for this blueprint. This must match the application ID used - when initiating rerun for any data logging you wish to associate with this blueprint. - port: - The port to listen on. - memory_limit: - An upper limit on how much memory the Rerun Viewer should use. - When this limit is reached, Rerun will drop the oldest data. - Example: `16GB` or `50%` (of system total). - hide_welcome_screen: - Hide the normal Rerun welcome screen. - - """ - _spawn_viewer(port=port, memory_limit=memory_limit, hide_welcome_screen=hide_welcome_screen) - self.connect(application_id=application_id, url=f"http://127.0.0.1:{port}") - - def spawn_grpc( - self, application_id: str, port: int = 9876, memory_limit: str = "75%", hide_welcome_screen: bool = False - ) -> None: - """ - Spawn a Rerun viewer with this blueprint. - Parameters ---------- application_id: diff --git a/rerun_py/rerun_sdk/rerun/sinks.py b/rerun_py/rerun_sdk/rerun/sinks.py index 7af4695abc02..7d0cb7a26b77 100644 --- a/rerun_py/rerun_sdk/rerun/sinks.py +++ b/rerun_py/rerun_sdk/rerun/sinks.py @@ -501,57 +501,6 @@ def spawn( """ - return spawn_grpc( - port=port, - connect=connect, - memory_limit=memory_limit, - hide_welcome_screen=hide_welcome_screen, - default_blueprint=default_blueprint, - recording=recording, # NOLINT: conversion not needed - ) - - -def spawn_grpc( - *, - port: int = 9876, - connect: bool = True, - memory_limit: str = "75%", - hide_welcome_screen: bool = False, - default_blueprint: BlueprintLike | None = None, - recording: RecordingStream | None = None, -) -> None: - """ - Spawn a Rerun Viewer, listening on the given port. - - This is often the easiest and best way to use Rerun. - Just call this once at the start of your program. - - You can also call [rerun.init][] with a `spawn=True` argument. - - Parameters - ---------- - port: - The port to listen on. - connect: - also connect to the viewer and stream logging data to it. - memory_limit: - An upper limit on how much memory the Rerun Viewer should use. - When this limit is reached, Rerun will drop the oldest data. - Example: `16GB` or `50%` (of system total). - hide_welcome_screen: - Hide the normal Rerun welcome screen. - recording: - Specifies the [`rerun.RecordingStream`][] to use if `connect = True`. - If left unspecified, defaults to the current active data recording, if there is one. - See also: [`rerun.init`][], [`rerun.set_global_data_recording`][]. - default_blueprint - Optionally set a default blueprint to use for this application. If the application - already has an active blueprint, the new blueprint won't become active until the user - clicks the "reset blueprint" button. If you want to activate the new blueprint - immediately, instead use the [`rerun.send_blueprint`][] API. - - """ - if not is_recording_enabled(recording): logging.warning("Rerun is disabled - spawn() call ignored.") return From 274dce38e95a2b5efcc9cd47843f01442b6ae6b0 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Thu, 30 Jan 2025 23:52:31 +0100 Subject: [PATCH 61/87] Remove mention of `TcpSink` --- crates/top/re_sdk/src/recording_stream.rs | 18 ++---------------- tests/cpp/plot_dashboard_stress/main.cpp | 4 ++-- 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/crates/top/re_sdk/src/recording_stream.rs b/crates/top/re_sdk/src/recording_stream.rs index 366da29d6d0c..8d7cd942c211 100644 --- a/crates/top/re_sdk/src/recording_stream.rs +++ b/crates/top/re_sdk/src/recording_stream.rs @@ -336,10 +336,6 @@ impl RecordingStreamBuilder { /// Creates a new [`RecordingStream`] that is pre-configured to stream the data through to a /// remote Rerun instance. /// - /// `flush_timeout` is the minimum time the [`TcpSink`][`crate::log_sink::TcpSink`] will - /// wait during a flush before potentially dropping data. Note: Passing `None` here can cause a - /// call to `flush` to block indefinitely if a connection cannot be established. - /// /// ## Example /// /// ```no_run @@ -360,10 +356,6 @@ impl RecordingStreamBuilder { /// Creates a new [`RecordingStream`] that is pre-configured to stream the data through to a /// remote Rerun instance. /// - /// `flush_timeout` is the minimum time the [`TcpSink`][`crate::log_sink::TcpSink`] will - /// wait during a flush before potentially dropping data. Note: Passing `None` here can cause a - /// call to `flush` to block indefinitely if a connection cannot be established. - /// /// ## Example /// /// ```no_run @@ -1714,8 +1706,7 @@ impl RecordingStream { } impl RecordingStream { - /// Swaps the underlying sink for a [`crate::log_sink::TcpSink`] sink pre-configured to use - /// the specified address. + /// Swaps the underlying sink for a sink pre-configured to use the specified address. /// /// See also [`Self::connect_opts`] if you wish to configure the TCP connection. /// @@ -1727,12 +1718,7 @@ impl RecordingStream { self.connect_grpc(); } - /// Swaps the underlying sink for a [`crate::log_sink::TcpSink`] sink pre-configured to use - /// the specified address. - /// - /// `flush_timeout` is the minimum time the [`TcpSink`][`crate::log_sink::TcpSink`] will - /// wait during a flush before potentially dropping data. Note: Passing `None` here can cause a - /// call to `flush` to block indefinitely if a connection cannot be established. + /// Swaps the underlying sink for a sink pre-configured to use the specified address. /// /// This is a convenience wrapper for [`Self::set_sink`] that upholds the same guarantees in /// terms of data durability and ordering. diff --git a/tests/cpp/plot_dashboard_stress/main.cpp b/tests/cpp/plot_dashboard_stress/main.cpp index e8b26c35b0d4..11c2f3328ed0 100644 --- a/tests/cpp/plot_dashboard_stress/main.cpp +++ b/tests/cpp/plot_dashboard_stress/main.cpp @@ -57,7 +57,7 @@ int main(int argc, char** argv) { // TODO(#4602): need common rerun args helper library if (args["spawn"].as()) { - rec.spawn_grpc().exit_on_failure(); + rec.spawn().exit_on_failure(); } else if (args["connect"].as()) { rec.connect_grpc().exit_on_failure(); } else if (args["stdout"].as()) { @@ -65,7 +65,7 @@ int main(int argc, char** argv) { } else if (args.count("save")) { rec.save(args["save"].as()).exit_on_failure(); } else { - rec.spawn_grpc().exit_on_failure(); + rec.spawn().exit_on_failure(); } const auto num_plots = args["num-plots"].as(); From ce965dd844f95b1453d3458e8b19b205b8006af3 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Thu, 30 Jan 2025 23:53:39 +0100 Subject: [PATCH 62/87] Fix `connect` usage --- crates/top/rerun/src/clap.rs | 2 +- examples/rust/custom_callback/src/app.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/top/rerun/src/clap.rs b/crates/top/rerun/src/clap.rs index 960263ab243d..d8a9966df2e6 100644 --- a/crates/top/rerun/src/clap.rs +++ b/crates/top/rerun/src/clap.rs @@ -111,7 +111,7 @@ impl RerunArgs { )), RerunBehavior::Connect(url) => Ok(( - RecordingStreamBuilder::new(application_id).connect_opts(url)?, + RecordingStreamBuilder::new(application_id).connect_grpc_opts(url)?, Default::default(), )), diff --git a/examples/rust/custom_callback/src/app.rs b/examples/rust/custom_callback/src/app.rs index 99ce71f7a9af..4af116efc08b 100644 --- a/examples/rust/custom_callback/src/app.rs +++ b/examples/rust/custom_callback/src/app.rs @@ -15,7 +15,7 @@ use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; async fn main() -> Result<(), Box> { let mut app = ControlApp::bind("127.0.0.1:8888").await?.run(); let rec = rerun::RecordingStreamBuilder::new("rerun_example_custom_callback") - .connect_opts("http://127.0.0.1:9877")?; + .connect_grpc_opts("http://127.0.0.1:9877")?; // Add a handler for incoming messages let add_rec = rec.clone(); From 34531cc40da85ab8e038c3a085362a9406ace7a8 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Thu, 30 Jan 2025 23:57:31 +0100 Subject: [PATCH 63/87] Exclude fake link --- lychee.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lychee.toml b/lychee.toml index 6ee47a9adf95..404e1bc15eab 100644 --- a/lychee.toml +++ b/lychee.toml @@ -116,6 +116,9 @@ exclude = [ 'https://your-hosted-asset-url.com/widget.js', 'file:///path/to/file', 'rerun://localhost:51234/recording/some-recording-id', + 'http://foo.zip', + 'https://foo.zip', + 'https://redap.rerun.io', # Link fragments and data links in examples. 'https://raw.githubusercontent.com/googlefonts/noto-emoji/', # URL fragment. From 2cdd6579d267895736ca4d419c23371334507961 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Fri, 31 Jan 2025 00:14:12 +0100 Subject: [PATCH 64/87] Update sdk with default port/url Also bring back `default_server_addr` and `default_flush_timeout` --- crates/top/re_sdk/src/lib.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/crates/top/re_sdk/src/lib.rs b/crates/top/re_sdk/src/lib.rs index ad7242bc0482..449a61011e24 100644 --- a/crates/top/re_sdk/src/lib.rs +++ b/crates/top/re_sdk/src/lib.rs @@ -33,6 +33,28 @@ pub use self::recording_stream::{ RecordingStreamResult, }; +/// The default potr of a Rerun gRPC server. +pub const DEFAULT_SERVER_PORT: u16 = 9876; + +/// The default URL of a Rerun gRPC server. +/// +/// This isn't used to _host_ the server, only to _connect_ to it. +pub const DEFAULT_CONNECT_URL: &str = "http://127.0.0.1:9876"; + +/// The default address of a Rerun TCP server which an SDK connects to. +#[deprecated(since = "0.22.0", note = "migrate to connect_grpc")] +pub fn default_server_addr() -> std::net::SocketAddr { + std::net::SocketAddr::from(([127, 0, 0, 1], DEFAULT_SERVER_PORT)) +} + +/// The default amount of time to wait for the TCP connection to resume during a flush +#[allow(clippy::unnecessary_wraps)] +#[deprecated(since = "0.22.0", note = "flush timeout no longer has any effect")] +pub fn default_flush_timeout() -> Option { + // NOTE: This is part of the SDK and meant to be used where we accept `Option` values. + Some(std::time::Duration::from_secs(2)) +} + pub use re_log_types::{ entity_path, ApplicationId, EntityPath, EntityPathPart, Instance, StoreId, StoreKind, }; From 51c3c1076a4874535793e49a00d5dcb51bd0f585 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Fri, 31 Jan 2025 00:14:53 +0100 Subject: [PATCH 65/87] Fix manual assert --- crates/store/re_data_source/src/data_source.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/store/re_data_source/src/data_source.rs b/crates/store/re_data_source/src/data_source.rs index 985ce46d9024..6396bf8b3c24 100644 --- a/crates/store/re_data_source/src/data_source.rs +++ b/crates/store/re_data_source/src/data_source.rs @@ -314,7 +314,5 @@ fn test_data_source_from_uri() { } } - if failed { - panic!("one or more test cases failed"); - } + assert!(!failed, "one or more test cases failed"); } From 6c9d47c49c7ac2028a65d7030a5842654103f683 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Fri, 31 Jan 2025 00:38:27 +0100 Subject: [PATCH 66/87] Add deprecation warning for `rr_recording_stream_connect` --- rerun_cpp/src/rerun/c/compiler_utils.h | 10 ++++++++++ rerun_cpp/src/rerun/c/rerun.h | 4 +++- rerun_cpp/src/rerun/recording_stream.cpp | 7 +++++++ rerun_cpp/tests/recording_stream.cpp | 2 ++ 4 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 rerun_cpp/src/rerun/c/compiler_utils.h diff --git a/rerun_cpp/src/rerun/c/compiler_utils.h b/rerun_cpp/src/rerun/c/compiler_utils.h new file mode 100644 index 000000000000..d57ae5b36df0 --- /dev/null +++ b/rerun_cpp/src/rerun/c/compiler_utils.h @@ -0,0 +1,10 @@ +#ifndef RR_DEPRECATED +// Mark as deprecated in C +#if defined(__GNUC__) || defined(__clang__) +#define RR_DEPRECATED(msg) __attribute__((deprecated)) +#elif defined(_MSC_VER) +#define RR_DEPRECATED(msg) __declspec(deprecated(msg)) +#else +#define RR_DEPRECATED(msg) +#endif // define checks +#endif // RR_DEPRECATED diff --git a/rerun_cpp/src/rerun/c/rerun.h b/rerun_cpp/src/rerun/c/rerun.h index ee12ee3eecea..6b4b3adf3f71 100644 --- a/rerun_cpp/src/rerun/c/rerun.h +++ b/rerun_cpp/src/rerun/c/rerun.h @@ -17,6 +17,7 @@ extern "C" { #include #include #include "arrow_c_data_interface.h" +#include "compiler_utils.h" #include "sdk_info.h" // ---------------------------------------------------------------------------- @@ -423,9 +424,10 @@ extern bool rr_recording_stream_is_enabled(rr_recording_stream stream, rr_error* /// /// This function returns immediately and will only raise an error for argument parsing errors, /// not for connection errors as these happen asynchronously. +RR_DEPRECATED("use rr_recording_stream_connect_grpc instead") extern void rr_recording_stream_connect( rr_recording_stream stream, rr_string tcp_addr, float flush_timeout_sec, rr_error* error -) __attribute__((deprecated)); +); /// Connect to a remote Rerun Viewer on the given HTTP(S) URL. /// diff --git a/rerun_cpp/src/rerun/recording_stream.cpp b/rerun_cpp/src/rerun/recording_stream.cpp index 184f1d32b52a..19d4d43b0332 100644 --- a/rerun_cpp/src/rerun/recording_stream.cpp +++ b/rerun_cpp/src/rerun/recording_stream.cpp @@ -104,19 +104,26 @@ namespace rerun { } } + RR_PUSH_WARNINGS + RR_DISABLE_DEPRECATION_WARNING Error RecordingStream::connect(std::string_view tcp_addr, float flush_timeout_sec) const { return RecordingStream::connect_tcp(tcp_addr, flush_timeout_sec); } + RR_POP_WARNINGS + Error RecordingStream::connect_tcp(std::string_view tcp_addr, float flush_timeout_sec) const { rr_error status = {}; + RR_PUSH_WARNINGS + RR_DISABLE_DEPRECATION_WARNING rr_recording_stream_connect( _id, detail::to_rr_string(tcp_addr), flush_timeout_sec, &status ); + RR_POP_WARNINGS return status; } diff --git a/rerun_cpp/tests/recording_stream.cpp b/rerun_cpp/tests/recording_stream.cpp index 6f4a1151eec0..beba3c877609 100644 --- a/rerun_cpp/tests/recording_stream.cpp +++ b/rerun_cpp/tests/recording_stream.cpp @@ -298,6 +298,7 @@ SCENARIO("RecordingStream can log to file", TEST_TAG) { } void test_logging_to_connection(const char* address, const rerun::RecordingStream& stream) { + RR_PUSH_WARNINGS RR_DISABLE_DEPRECATION_WARNING // TODO(jan): Remove once `connect` is removed { // We changed to taking std::string_view instead of const char* and constructing such from nullptr crashes @@ -341,6 +342,7 @@ void test_logging_to_connection(const char* address, const rerun::RecordingStrea } } } + RR_POP_WARNINGS } SCENARIO("RecordingStream can connect", TEST_TAG) { From 357f243f0442e78d6acaf26f802dd24a2319ffc8 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Fri, 31 Jan 2025 00:44:29 +0100 Subject: [PATCH 67/87] Try to fix cpp formatting --- rerun_cpp/tests/recording_stream.cpp | 84 ++++++++++++++-------------- 1 file changed, 43 insertions(+), 41 deletions(-) diff --git a/rerun_cpp/tests/recording_stream.cpp b/rerun_cpp/tests/recording_stream.cpp index beba3c877609..9aaca7105eaa 100644 --- a/rerun_cpp/tests/recording_stream.cpp +++ b/rerun_cpp/tests/recording_stream.cpp @@ -297,54 +297,56 @@ SCENARIO("RecordingStream can log to file", TEST_TAG) { } } -void test_logging_to_connection(const char* address, const rerun::RecordingStream& stream) { - RR_PUSH_WARNINGS - RR_DISABLE_DEPRECATION_WARNING // TODO(jan): Remove once `connect` is removed - { - // We changed to taking std::string_view instead of const char* and constructing such from nullptr crashes - // at least on some C++ implementations. - // If we'd want to support this in earnest we'd have to create out own string_view type. - // - // AND_GIVEN("a nullptr for the socket address") { - // THEN("then the connect call returns a null argument error") { - // CHECK(stream.connect(nullptr, 0.0f).code == rerun::ErrorCode::UnexpectedNullArgument); - // } - // } - AND_GIVEN("an invalid address for the socket address") { - THEN("connect call fails") { - CHECK( - stream.connect("definitely not valid!").code == - rerun::ErrorCode::InvalidSocketAddress - ); - } +RR_PUSH_WARNINGS +RR_DISABLE_DEPRECATION_WARNING // TODO(jan): Remove once `connect` is removed + void + test_logging_to_connection( + const char* address, const rerun::RecordingStream& stream + ) { // We changed to taking std::string_view instead of const char* and constructing such from nullptr crashes + // at least on some C++ implementations. + // If we'd want to support this in earnest we'd have to create out own string_view type. + // + // AND_GIVEN("a nullptr for the socket address") { + // THEN("then the connect call returns a null argument error") { + // CHECK(stream.connect(nullptr, 0.0f).code == rerun::ErrorCode::UnexpectedNullArgument); + // } + // } + AND_GIVEN("an invalid address for the socket address") { + THEN("connect call fails") { + CHECK( + stream.connect("definitely not valid!").code == + rerun::ErrorCode::InvalidSocketAddress + ); } - AND_GIVEN("a valid socket address " << address) { - THEN("connect call returns no error") { - CHECK(stream.connect(address).code == rerun::ErrorCode::Ok); - - WHEN("logging an archetype and then flushing") { - check_logged_error([&] { - stream.log( - "archetype", - rerun::Points2D({ - rerun::Vec2D{1.0, 2.0}, - rerun::Vec2D{4.0, 5.0}, - }) - ); - }); - - stream.flush_blocking(); - - THEN("does not crash") { - // No easy way to see if it got sent. - } + } + + AND_GIVEN("a valid socket address " << address) { + THEN("connect call returns no error") { + CHECK(stream.connect(address).code == rerun::ErrorCode::Ok); + + WHEN("logging an archetype and then flushing") { + check_logged_error([&] { + stream.log( + "archetype", + rerun::Points2D({ + rerun::Vec2D{1.0, 2.0}, + rerun::Vec2D{4.0, 5.0}, + }) + ); + }); + + stream.flush_blocking(); + + THEN("does not crash") { + // No easy way to see if it got sent. } } } } - RR_POP_WARNINGS } +RR_POP_WARNINGS + SCENARIO("RecordingStream can connect", TEST_TAG) { const char* address = "127.0.0.1:9876"; GIVEN("a new RecordingStream") { From fbd79fda765d4886f892a2048f9bca266f5069ab Mon Sep 17 00:00:00 2001 From: jprochazk Date: Tue, 4 Feb 2025 16:26:42 +0100 Subject: [PATCH 68/87] temp --- .../re_grpc_client/src/message_proxy/write.rs | 4 +- crates/store/re_grpc_server/src/lib.rs | 1 + crates/top/rerun-cli/src/bin/rerun.rs | 3 +- crates/top/rerun/src/commands/entrypoint.rs | 52 +++++++++---------- 4 files changed, 29 insertions(+), 31 deletions(-) diff --git a/crates/store/re_grpc_client/src/message_proxy/write.rs b/crates/store/re_grpc_client/src/message_proxy/write.rs index 26333f01247c..cbd0eb731d19 100644 --- a/crates/store/re_grpc_client/src/message_proxy/write.rs +++ b/crates/store/re_grpc_client/src/message_proxy/write.rs @@ -141,7 +141,9 @@ async fn message_proxy_client( buffered_messages.push(msg); } Some(Cmd::Flush(tx)) => { - re_log::debug!("Flush requested"); + re_log::warn_once!( + "Attempted to flush while gRPC client was connecting." + ); if tx.send(()).is_err() { re_log::debug!("Failed to respond to flush: channel is closed"); return; diff --git a/crates/store/re_grpc_server/src/lib.rs b/crates/store/re_grpc_server/src/lib.rs index 8e35b594702e..37e0549dd3c7 100644 --- a/crates/store/re_grpc_server/src/lib.rs +++ b/crates/store/re_grpc_server/src/lib.rs @@ -134,6 +134,7 @@ pub async fn serve_with_send( } } +/// Spawn the server and subscribe to its message queue. pub fn spawn_with_recv( addr: SocketAddr, memory_limit: MemoryLimit, diff --git a/crates/top/rerun-cli/src/bin/rerun.rs b/crates/top/rerun-cli/src/bin/rerun.rs index 022a7e4042b2..51cbb76f610f 100644 --- a/crates/top/rerun-cli/src/bin/rerun.rs +++ b/crates/top/rerun-cli/src/bin/rerun.rs @@ -16,8 +16,7 @@ use re_memory::AccountingAllocator; static GLOBAL: AccountingAllocator = AccountingAllocator::new(mimalloc::MiMalloc); -#[tokio::main] -async fn main() -> std::process::ExitCode { +fn main() -> std::process::ExitCode { let main_thread_token = rerun::MainThreadToken::i_promise_i_am_on_the_main_thread(); re_log::setup_logging(); diff --git a/crates/top/rerun/src/commands/entrypoint.rs b/crates/top/rerun/src/commands/entrypoint.rs index 3b5c81a134e8..cbb5c0d368bc 100644 --- a/crates/top/rerun/src/commands/entrypoint.rs +++ b/crates/top/rerun/src/commands/entrypoint.rs @@ -2,6 +2,7 @@ use std::net::IpAddr; use clap::{CommandFactory, Subcommand}; use itertools::Itertools; +use tokio::runtime::Runtime; use re_data_source::DataSource; use re_log_types::LogMsg; @@ -157,6 +158,12 @@ When persisted, the state will be stored at the following locations: #[clap(long)] serve_web: bool, + /// Do not attempt to start a new server, instead try to connect to an existing one. + /// + /// Optionally accepts an HTTP(S) URL to a gRPC server. + #[clap(long)] + connect: Option>, + /// This is a hint that we expect a recording to stream in very soon. /// /// This is set by the `spawn()` method in our logging SDK. @@ -587,7 +594,7 @@ where } } } else { - run_in_tokio(main_thread_token, build_info, call_source, args) + run_impl(main_thread_token, build_info, call_source, args) }; match res { @@ -609,29 +616,6 @@ where } } -/// Ensures that we are running in the context of a tokio runtime. -fn run_in_tokio( - main_thread_token: crate::MainThreadToken, - build_info: re_build_info::BuildInfo, - call_source: CallSource, - args: Args, -) -> anyhow::Result<()> { - // tokio is a hard dependency as of our gRPC migration, - // so we must ensure it is always available: - if let Ok(handle) = tokio::runtime::Handle::try_current() { - // This thread already has a tokio runtime. - let _guard = handle.enter(); - run_impl(main_thread_token, build_info, call_source, args) - } else { - // We don't have a runtime yet, create one now. - let mut builder = tokio::runtime::Builder::new_multi_thread(); - builder.enable_all(); - let rt = builder.build()?; - let _guard = rt.enter(); - run_impl(main_thread_token, build_info, call_source, args) - } -} - fn run_impl( _main_thread_token: crate::MainThreadToken, _build_info: re_build_info::BuildInfo, @@ -640,7 +624,11 @@ fn run_impl( ) -> anyhow::Result<()> { #[cfg(feature = "native_viewer")] let profiler = run_profiler(&args); - let mut is_another_viewer_running = false; + let mut is_another_server_running = false; + + // NOTE: Be careful about spawning tasks here! We don't want the runtime to run on the main thread, + // as we need that one for our UI. Use `rt.enter()` or `rt.spawn()` if you need to spawn a task. + let rt = Runtime::new()?; #[cfg(feature = "native_viewer")] let startup_options = { @@ -718,13 +706,19 @@ fn run_impl( } } + // We may need to spawn tasks from this point on: + let _guard = rt.enter(); let mut rxs = data_sources .into_iter() .map(|data_source| data_source.stream(None)) .collect::, _>>()?; #[cfg(feature = "server")] - { + if let Some(url) = args.connect { + let url = url.unwrap_or_else(|| format!("http://{server_addr}")); + let rx = re_sdk::external::re_grpc_client::message_proxy::stream(&url, None)?; + rxs.push(rx); + } else { // Check if there is already a viewer running and if so, send the data to it. use std::net::TcpStream; if TcpStream::connect_timeout(&server_addr, std::time::Duration::from_secs(1)).is_ok() { @@ -732,7 +726,7 @@ fn run_impl( %server_addr, "A process is already listening at this address. Assuming it's a Rerun Viewer." ); - is_another_viewer_running = true; + is_another_server_running = true; } else { let server: Receiver = re_grpc_server::spawn_with_recv( server_addr, @@ -803,13 +797,15 @@ fn run_impl( } Ok(()) - } else if is_another_viewer_running { + } else if is_another_server_running { // Another viewer is already running on the specified address let url = format!("http://{server_addr}") .parse() .expect("should always be valid"); re_log::info!(%url, "Another viewer is already running, streaming data to it."); + // This spawns its own single-threaded runtime on a separate thread, + // no need to `rt.enter()`: let sink = re_sdk::sink::GrpcSink::new(url); for rx in rxs { From e7eb688c43b747eb40deb09f5e3860a27fddf801 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Mon, 10 Feb 2025 00:06:55 +0100 Subject: [PATCH 69/87] join parse url tests --- Cargo.lock | 106 ++---------------- .../re_grpc_client/src/message_proxy/read.rs | 43 ++++--- 2 files changed, 33 insertions(+), 116 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d409d296c783..54cf5756eeed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1740,6 +1740,7 @@ dependencies = [ "parking_lot", "re_crash_handler", "re_error", + "re_grpc_server", "rerun", "serde", "tokio", @@ -1767,8 +1768,9 @@ version = "0.23.0-alpha.1+dev" dependencies = [ "mimalloc", "re_crash_handler", - "re_sdk_comms", + "re_grpc_server", "re_viewer", + "tokio", ] [[package]] @@ -1805,12 +1807,6 @@ dependencies = [ "syn 2.0.87", ] -[[package]] -name = "data-encoding" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" - [[package]] name = "data-url" version = "0.3.1" @@ -2405,29 +2401,15 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "ewebsock" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "679247b4a005c82218a5f13b713239b0b6d484ec25347a719f5b7066152a748a" -dependencies = [ - "document-features", - "js-sys", - "log", - "tungstenite", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - [[package]] name = "extend_viewer_ui" version = "0.23.0-alpha.1+dev" dependencies = [ "mimalloc", "re_crash_handler", - "re_sdk_comms", + "re_grpc_server", "re_viewer", + "tokio", ] [[package]] @@ -5996,7 +5978,6 @@ dependencies = [ "re_log_types", "re_smart_channel", "re_tracing", - "re_ws_comms", ] [[package]] @@ -6172,6 +6153,7 @@ dependencies = [ name = "re_grpc_server" version = "0.23.0-alpha.1+dev" dependencies = [ + "parking_lot", "re_build_info", "re_byte_size", "re_chunk", @@ -6181,6 +6163,7 @@ dependencies = [ "re_log_types", "re_memory", "re_protos", + "re_smart_channel", "re_tracing", "re_types", "tokio", @@ -6513,36 +6496,20 @@ dependencies = [ "re_chunk_store", "re_data_loader", "re_grpc_client", + "re_grpc_server", "re_log", "re_log_encoding", "re_log_types", "re_memory", - "re_sdk_comms", "re_smart_channel", "re_types_core", "re_web_viewer_server", - "re_ws_comms", "similar-asserts", "thiserror 1.0.65", + "tokio", "webbrowser", ] -[[package]] -name = "re_sdk_comms" -version = "0.23.0-alpha.1+dev" -dependencies = [ - "ahash", - "crossbeam", - "document-features", - "rand", - "re_build_info", - "re_log", - "re_log_encoding", - "re_log_types", - "re_smart_channel", - "thiserror 1.0.65", -] - [[package]] name = "re_selection_panel" version = "0.23.0-alpha.1+dev" @@ -7111,7 +7078,6 @@ dependencies = [ "re_memory", "re_query", "re_renderer", - "re_sdk_comms", "re_selection_panel", "re_smart_channel", "re_time_panel", @@ -7132,7 +7098,6 @@ dependencies = [ "re_viewer_context", "re_viewport", "re_viewport_blueprint", - "re_ws_comms", "rfd", "ron", "serde", @@ -7268,26 +7233,6 @@ dependencies = [ "tiny_http", ] -[[package]] -name = "re_ws_comms" -version = "0.23.0-alpha.1+dev" -dependencies = [ - "anyhow", - "bincode", - "document-features", - "ewebsock", - "parking_lot", - "polling", - "re_format", - "re_log", - "re_log_types", - "re_memory", - "re_smart_channel", - "re_tracing", - "thiserror 1.0.65", - "tungstenite", -] - [[package]] name = "redox_syscall" version = "0.4.1" @@ -7492,20 +7437,20 @@ dependencies = [ "re_error", "re_format", "re_format_arrow", + "re_grpc_server", "re_log", "re_log_encoding", "re_log_types", "re_memory", "re_sdk", - "re_sdk_comms", "re_smart_channel", "re_tracing", "re_types", "re_video", "re_viewer", "re_web_viewer_server", - "re_ws_comms", "similar-asserts", + "tokio", "unindent", ] @@ -7573,6 +7518,7 @@ dependencies = [ "re_chunk_store", "re_dataframe", "re_grpc_client", + "re_grpc_server", "re_log", "re_log_encoding", "re_log_types", @@ -7582,7 +7528,6 @@ dependencies = [ "re_sorbet", "re_video", "re_web_viewer_server", - "re_ws_comms", "tokio", "tokio-stream", "tonic", @@ -9316,27 +9261,6 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5902c5d130972a0000f60860bfbf46f7ca3db5391eddfedd1b8728bd9dc96c0e" -[[package]] -name = "tungstenite" -version = "0.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" -dependencies = [ - "byteorder", - "bytes", - "data-encoding", - "http 1.1.0", - "httparse", - "log", - "rand", - "rustls 0.23.18", - "rustls-pki-types", - "sha1", - "thiserror 1.0.65", - "utf-8", - "webpki-roots 0.26.6", -] - [[package]] name = "twox-hash" version = "1.6.3" @@ -9507,12 +9431,6 @@ dependencies = [ "tiny-skia-path", ] -[[package]] -name = "utf-8" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" - [[package]] name = "utf16_iter" version = "1.0.5" diff --git a/crates/store/re_grpc_client/src/message_proxy/read.rs b/crates/store/re_grpc_client/src/message_proxy/read.rs index c9b0405edb4f..765c10acaccf 100644 --- a/crates/store/re_grpc_client/src/message_proxy/read.rs +++ b/crates/store/re_grpc_client/src/message_proxy/read.rs @@ -158,27 +158,26 @@ async fn stream_async( mod tests { use super::*; - macro_rules! test_parse_url { - ($name:ident, $url:literal, error) => { - #[test] - fn $name() { - assert!(MessageProxyUrl::parse($url).is_err()); - } - }; - - ($name:ident, $url:literal, expected: $expected_http:literal) => { - #[test] - fn $name() { - assert_eq!( - MessageProxyUrl::parse($url).map(|v| v.to_http()), - Ok($expected_http.to_owned()) - ); - } - }; + #[test] + fn test_parse_url() { + struct Case { + input: &'static str, + expected: &'static str, + } + let cases = [ + Case { + input: "temp://127.0.0.1:9876", + expected: "http://127.0.0.1:9876", + }, + Case { + input: "http://127.0.0.1:9876", + expected: "http://127.0.0.1:9876", + }, + ]; + + for Case { input, expected } in cases { + let actual = MessageProxyUrl::parse(input).map(|v| v.to_http()); + assert_eq!(actual, Ok(expected.to_owned())); + } } - - test_parse_url!(basic_temp, "temp://127.0.0.1:9876", expected: "http://127.0.0.1:9876"); - // TODO(#8761): URL prefix - test_parse_url!(basic_http, "http://127.0.0.1:9876", expected: "http://127.0.0.1:9876"); - test_parse_url!(invalid, "definitely not valid", error); } From 293415eda48f16bd5437735034f1e3b9c56955a4 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Mon, 10 Feb 2025 00:20:13 +0100 Subject: [PATCH 70/87] fix lint It's probably better to leave it as Option>, coming up with a separate type would also mean implemeting a parser for it, adding a bunch of unnecessary boilerplate. --- crates/top/rerun/src/commands/entrypoint.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/top/rerun/src/commands/entrypoint.rs b/crates/top/rerun/src/commands/entrypoint.rs index cbb5c0d368bc..775a0b52c9c1 100644 --- a/crates/top/rerun/src/commands/entrypoint.rs +++ b/crates/top/rerun/src/commands/entrypoint.rs @@ -162,6 +162,7 @@ When persisted, the state will be stored at the following locations: /// /// Optionally accepts an HTTP(S) URL to a gRPC server. #[clap(long)] + #[allow(clippy::option_option)] // Tri-state: none, --connect, --connect . connect: Option>, /// This is a hint that we expect a recording to stream in very soon. From 1d32363febfab7e69f5e5629a31ad54a016aa338 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Mon, 10 Feb 2025 01:12:26 +0100 Subject: [PATCH 71/87] remove unused dep --- Cargo.lock | 1 - crates/top/rerun-cli/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 54cf5756eeed..1ab74e8a0691 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7466,7 +7466,6 @@ dependencies = [ "re_log", "re_memory", "rerun", - "tokio", ] [[package]] diff --git a/crates/top/rerun-cli/Cargo.toml b/crates/top/rerun-cli/Cargo.toml index 80c983bb38e4..1725929c6a36 100644 --- a/crates/top/rerun-cli/Cargo.toml +++ b/crates/top/rerun-cli/Cargo.toml @@ -99,7 +99,6 @@ rerun = { workspace = true, features = [ document-features.workspace = true mimalloc = "0.1.43" -tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } [build-dependencies] From 04a8e774bb8a70466edce8bec061930770a6749c Mon Sep 17 00:00:00 2001 From: jprochazk Date: Mon, 10 Feb 2025 01:26:26 +0100 Subject: [PATCH 72/87] dont fail on first test case --- crates/store/re_grpc_client/src/message_proxy/read.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/store/re_grpc_client/src/message_proxy/read.rs b/crates/store/re_grpc_client/src/message_proxy/read.rs index 765c10acaccf..33bbb7f9bf6a 100644 --- a/crates/store/re_grpc_client/src/message_proxy/read.rs +++ b/crates/store/re_grpc_client/src/message_proxy/read.rs @@ -175,9 +175,14 @@ mod tests { }, ]; + let mut failed = false; for Case { input, expected } in cases { let actual = MessageProxyUrl::parse(input).map(|v| v.to_http()); - assert_eq!(actual, Ok(expected.to_owned())); + if actual != Ok(expected.to_owned()) { + eprintln!("expected {input:?} to parse as {expected:?}, got {actual:?} instead"); + failed = true; + } } + assert!(!failed, "one or more test cases failed"); } } From 0d8313fd44005dcd6d489f6f0d0d4e1f6cccad64 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Mon, 10 Feb 2025 13:28:27 +0100 Subject: [PATCH 73/87] do not gc blueprint Previously, all blueprint data, including activation commands, would be garbage collected. It doesn't make much sense, because blueprints use very little memory compared to the rest of a typical recording. --- crates/store/re_grpc_server/src/lib.rs | 125 ++++++++++++++++++++++--- 1 file changed, 110 insertions(+), 15 deletions(-) diff --git a/crates/store/re_grpc_server/src/lib.rs b/crates/store/re_grpc_server/src/lib.rs index 37e0549dd3c7..0de0b7b81b73 100644 --- a/crates/store/re_grpc_server/src/lib.rs +++ b/crates/store/re_grpc_server/src/lib.rs @@ -9,6 +9,7 @@ use std::pin::Pin; use re_byte_size::SizeBytes; use re_memory::MemoryLimit; use re_protos::{ + common::v0::StoreKind as StoreKindProto, log_msg::v0::LogMsg as LogMsgProto, sdk_comms::v0::{message_proxy_server, Empty}, }; @@ -278,20 +279,31 @@ impl EventLoop { return; }; + // We put store info, blueprint data, and blueprint activation commands + // in a separate queue that does *not* get garbage collected. use re_protos::log_msg::v0::log_msg::Msg; match inner { - // We consider `BlueprintActivationCommand` a temporal message, - // because it is sensitive to order, and it is safe to garbage collect - // if all the messages that came before it were also garbage collected, - // as it's the last message sent by the SDK when submitting a blueprint. - Msg::ArrowMsg(..) | Msg::BlueprintActivationCommand(..) => { + // Store info, blueprint activation commands + Msg::SetStoreInfo(..) | Msg::BlueprintActivationCommand(..) => { + self.persistent_message_queue.push_back(msg); + } + + // Blueprint data + Msg::ArrowMsg(ref inner) + if inner + .store_id + .as_ref() + .is_some_and(|id| id.kind() == StoreKindProto::Blueprint) => + { + self.persistent_message_queue.push_back(msg); + } + + // Recording data + Msg::ArrowMsg(..) => { let approx_size_bytes = message_size(&msg); self.ordered_message_bytes += approx_size_bytes; self.ordered_message_queue.push_back(msg); } - Msg::SetStoreInfo(..) => { - self.persistent_message_queue.push_back(msg); - } } } @@ -490,7 +502,7 @@ mod tests { /// Generates `n` log messages wrapped in a `SetStoreInfo` at the start and `BlueprintActivationCommand` at the end, /// to exercise message ordering. - fn fake_log_stream(n: usize) -> Vec { + fn fake_log_stream_blueprint(n: usize) -> Vec { let store_id = StoreId::random(StoreKind::Blueprint); let mut messages = Vec::new(); @@ -541,6 +553,47 @@ mod tests { messages } + fn fake_log_stream_recording(n: usize) -> Vec { + let store_id = StoreId::random(StoreKind::Recording); + + let mut messages = Vec::new(); + messages.push(LogMsg::SetStoreInfo(SetStoreInfo { + row_id: *RowId::new(), + info: StoreInfo { + application_id: ApplicationId("test".to_owned()), + store_id: store_id.clone(), + cloned_from: None, + is_official_example: true, + started: Time::now(), + store_source: StoreSource::RustSdk { + rustc_version: String::new(), + llvm_version: String::new(), + }, + store_version: Some(CrateVersion::LOCAL), + }, + })); + for _ in 0..n { + messages.push(LogMsg::ArrowMsg( + store_id.clone(), + re_chunk::Chunk::builder("test_entity".into()) + .with_archetype( + re_chunk::RowId::new(), + re_log_types::TimePoint::default().with( + re_log_types::Timeline::new_sequence("log_time"), + re_log_types::TimeInt::from_milliseconds(re_log_types::NonMinI64::MIN), + ), + &re_types::archetypes::Points2D::new([(0.0, 0.0), (1.0, 1.0), (2.0, 2.0)]), + ) + .build() + .unwrap() + .to_arrow_msg() + .unwrap(), + )); + } + + messages + } + async fn setup() -> (Completion, SocketAddr) { setup_with_memory_limit(MemoryLimit::UNLIMITED).await } @@ -599,7 +652,7 @@ mod tests { async fn pubsub_basic() { let (completion, addr) = setup().await; let mut client = make_client(addr).await; // We use the same client for both producing and consuming - let messages = fake_log_stream(3); + let messages = fake_log_stream_blueprint(3); // start reading let mut log_stream = client.read_messages(Empty {}).await.unwrap(); @@ -633,7 +686,7 @@ mod tests { async fn pubsub_history() { let (completion, addr) = setup().await; let mut client = make_client(addr).await; // We use the same client for both producing and consuming - let messages = fake_log_stream(3); + let messages = fake_log_stream_blueprint(3); // don't read anything yet - these messages should be sent to us as part of history when we call `read_messages` later @@ -662,7 +715,7 @@ mod tests { let (completion, addr) = setup().await; let mut producer = make_client(addr).await; // We use separate clients for producing and consuming let mut consumers = vec![make_client(addr).await, make_client(addr).await]; - let messages = fake_log_stream(3); + let messages = fake_log_stream_blueprint(3); // Initialize multiple read streams: let mut log_streams = vec![]; @@ -695,7 +748,7 @@ mod tests { let (completion, addr) = setup().await; let mut producers = vec![make_client(addr).await, make_client(addr).await]; let mut consumers = vec![make_client(addr).await, make_client(addr).await]; - let messages = fake_log_stream(3); + let messages = fake_log_stream_blueprint(3); // Initialize multiple read streams: let mut log_streams = vec![]; @@ -735,7 +788,7 @@ mod tests { // Use an absurdly low memory limit to force all messages to be dropped immediately from history let (completion, addr) = setup_with_memory_limit(MemoryLimit::from_bytes(1)).await; let mut client = make_client(addr).await; - let messages = fake_log_stream(3); + let messages = fake_log_stream_recording(3); // Write some messages client @@ -773,12 +826,54 @@ mod tests { completion.finish(); } + #[tokio::test] + async fn memory_limit_does_not_drop_blueprint() { + // Use an absurdly low memory limit to force all messages to be dropped immediately from history + let (completion, addr) = setup_with_memory_limit(MemoryLimit::from_bytes(1)).await; + let mut client = make_client(addr).await; + let messages = fake_log_stream_blueprint(3); + + // Write some messages + client + .write_messages(tokio_stream::iter( + messages + .clone() + .into_iter() + .map(|msg| log_msg_to_proto(msg, Compression::Off).unwrap()), + )) + .await + .unwrap(); + + // Start reading + let mut log_stream = client.read_messages(Empty {}).await.unwrap(); + let mut actual = vec![]; + loop { + let timeout_stream = log_stream.get_mut().timeout(Duration::from_millis(100)); + tokio::pin!(timeout_stream); + let timeout_result = timeout_stream.try_next().await; + match timeout_result { + Ok(Some(value)) => { + actual.push(log_msg_from_proto(value.unwrap()).unwrap()); + } + + // Stream closed | Timed out + Ok(None) | Err(_) => break, + } + } + + // The stream in this case only contains SetStoreInfo, ArrowMsg with StoreKind::Blueprint, + // and BlueprintActivationCommand. None of these things should be GC'd: + assert_eq!(messages, actual); + + completion.finish(); + } + #[tokio::test] async fn memory_limit_does_not_interrupt_stream() { // Use an absurdly low memory limit to force all messages to be dropped immediately from history let (completion, addr) = setup_with_memory_limit(MemoryLimit::from_bytes(1)).await; let mut client = make_client(addr).await; // We use the same client for both producing and consuming - let messages = fake_log_stream(3); + let messages = fake_log_stream_blueprint(3); // Start reading let mut log_stream = client.read_messages(Empty {}).await.unwrap(); From 9ba5ae395a0cb6139bbca5d137d4b3811925836f Mon Sep 17 00:00:00 2001 From: jprochazk Date: Mon, 10 Feb 2025 15:51:42 +0100 Subject: [PATCH 74/87] use proper response/request types instead of empty This way we can individually evolve those types. --- .../re_grpc_client/src/message_proxy/read.rs | 4 +- crates/store/re_grpc_server/src/lib.rs | 32 +++++++++----- .../re_protos/proto/rerun/v0/sdk_comms.proto | 7 +-- .../re_protos/src/v0/rerun.sdk_comms.v0.rs | 43 +++++++++++++------ 4 files changed, 58 insertions(+), 28 deletions(-) diff --git a/crates/store/re_grpc_client/src/message_proxy/read.rs b/crates/store/re_grpc_client/src/message_proxy/read.rs index 33bbb7f9bf6a..ace898f977b2 100644 --- a/crates/store/re_grpc_client/src/message_proxy/read.rs +++ b/crates/store/re_grpc_client/src/message_proxy/read.rs @@ -3,7 +3,7 @@ use std::fmt::Display; use re_log_encoding::protobuf_conversions::log_msg_from_proto; use re_log_types::LogMsg; use re_protos::sdk_comms::v0::message_proxy_client::MessageProxyClient; -use re_protos::sdk_comms::v0::Empty; +use re_protos::sdk_comms::v0::ReadMessagesRequest; use tokio_stream::StreamExt; use url::Url; @@ -120,7 +120,7 @@ async fn stream_async( re_log::debug!("Streaming messages from gRPC endpoint {url}"); let mut stream = client - .read_messages(Empty {}) + .read_messages(ReadMessagesRequest {}) .await .map_err(TonicStatusError)? .into_inner(); diff --git a/crates/store/re_grpc_server/src/lib.rs b/crates/store/re_grpc_server/src/lib.rs index 0de0b7b81b73..4be3ba84c756 100644 --- a/crates/store/re_grpc_server/src/lib.rs +++ b/crates/store/re_grpc_server/src/lib.rs @@ -11,7 +11,7 @@ use re_memory::MemoryLimit; use re_protos::{ common::v0::StoreKind as StoreKindProto, log_msg::v0::LogMsg as LogMsgProto, - sdk_comms::v0::{message_proxy_server, Empty}, + sdk_comms::v0::{message_proxy_server, ReadMessagesRequest, WriteMessagesResponse}, }; use tokio::net::TcpListener; use tokio::sync::broadcast; @@ -423,7 +423,7 @@ impl message_proxy_server::MessageProxy for MessageProxy { async fn write_messages( &self, request: tonic::Request>, - ) -> tonic::Result> { + ) -> tonic::Result> { let mut stream = request.into_inner(); loop { match stream.message().await { @@ -441,14 +441,14 @@ impl message_proxy_server::MessageProxy for MessageProxy { } } - Ok(tonic::Response::new(Empty {})) + Ok(tonic::Response::new(WriteMessagesResponse {})) } type ReadMessagesStream = LogMsgStream; async fn read_messages( &self, - _: tonic::Request, + _: tonic::Request, ) -> tonic::Result> { Ok(tonic::Response::new(self.new_client_stream().await)) } @@ -655,7 +655,7 @@ mod tests { let messages = fake_log_stream_blueprint(3); // start reading - let mut log_stream = client.read_messages(Empty {}).await.unwrap(); + let mut log_stream = client.read_messages(ReadMessagesRequest {}).await.unwrap(); // write a few messages client @@ -702,7 +702,7 @@ mod tests { .unwrap(); // Start reading now - we should receive full history at this point: - let mut log_stream = client.read_messages(Empty {}).await.unwrap(); + let mut log_stream = client.read_messages(ReadMessagesRequest {}).await.unwrap(); let actual = read_log_stream(&mut log_stream, messages.len()).await; assert_eq!(messages, actual); @@ -720,7 +720,12 @@ mod tests { // Initialize multiple read streams: let mut log_streams = vec![]; for consumer in &mut consumers { - log_streams.push(consumer.read_messages(Empty {}).await.unwrap()); + log_streams.push( + consumer + .read_messages(ReadMessagesRequest {}) + .await + .unwrap(), + ); } // Write a few messages using our single producer: @@ -753,7 +758,12 @@ mod tests { // Initialize multiple read streams: let mut log_streams = vec![]; for consumer in &mut consumers { - log_streams.push(consumer.read_messages(Empty {}).await.unwrap()); + log_streams.push( + consumer + .read_messages(ReadMessagesRequest {}) + .await + .unwrap(), + ); } // Write a few messages using each producer: @@ -802,7 +812,7 @@ mod tests { .unwrap(); // Start reading - let mut log_stream = client.read_messages(Empty {}).await.unwrap(); + let mut log_stream = client.read_messages(ReadMessagesRequest {}).await.unwrap(); let mut actual = vec![]; loop { let timeout_stream = log_stream.get_mut().timeout(Duration::from_millis(100)); @@ -845,7 +855,7 @@ mod tests { .unwrap(); // Start reading - let mut log_stream = client.read_messages(Empty {}).await.unwrap(); + let mut log_stream = client.read_messages(ReadMessagesRequest {}).await.unwrap(); let mut actual = vec![]; loop { let timeout_stream = log_stream.get_mut().timeout(Duration::from_millis(100)); @@ -876,7 +886,7 @@ mod tests { let messages = fake_log_stream_blueprint(3); // Start reading - let mut log_stream = client.read_messages(Empty {}).await.unwrap(); + let mut log_stream = client.read_messages(ReadMessagesRequest {}).await.unwrap(); // Write a few messages client diff --git a/crates/store/re_protos/proto/rerun/v0/sdk_comms.proto b/crates/store/re_protos/proto/rerun/v0/sdk_comms.proto index 27fc9c596673..fac6c5434fa9 100644 --- a/crates/store/re_protos/proto/rerun/v0/sdk_comms.proto +++ b/crates/store/re_protos/proto/rerun/v0/sdk_comms.proto @@ -17,8 +17,9 @@ import "rerun/v0/common.proto"; service MessageProxy { // TODO(jan): Would it be more efficient to send a "message batch" instead of individual messages? // It may allow us to amortize the overhead of the gRPC protocol. - rpc WriteMessages(stream rerun.log_msg.v0.LogMsg) returns (Empty) {} - rpc ReadMessages(Empty) returns (stream rerun.log_msg.v0.LogMsg) {} + rpc WriteMessages(stream rerun.log_msg.v0.LogMsg) returns (WriteMessagesResponse) {} + rpc ReadMessages(ReadMessagesRequest) returns (stream rerun.log_msg.v0.LogMsg) {} } -message Empty {} +message WriteMessagesResponse {} +message ReadMessagesRequest {} diff --git a/crates/store/re_protos/src/v0/rerun.sdk_comms.v0.rs b/crates/store/re_protos/src/v0/rerun.sdk_comms.v0.rs index ac4c4e6082f4..f4c8233641e6 100644 --- a/crates/store/re_protos/src/v0/rerun.sdk_comms.v0.rs +++ b/crates/store/re_protos/src/v0/rerun.sdk_comms.v0.rs @@ -1,14 +1,26 @@ // This file is @generated by prost-build. #[derive(Clone, Copy, PartialEq, ::prost::Message)] -pub struct Empty {} -impl ::prost::Name for Empty { - const NAME: &'static str = "Empty"; +pub struct WriteMessagesResponse {} +impl ::prost::Name for WriteMessagesResponse { + const NAME: &'static str = "WriteMessagesResponse"; const PACKAGE: &'static str = "rerun.sdk_comms.v0"; fn full_name() -> ::prost::alloc::string::String { - "rerun.sdk_comms.v0.Empty".into() + "rerun.sdk_comms.v0.WriteMessagesResponse".into() } fn type_url() -> ::prost::alloc::string::String { - "/rerun.sdk_comms.v0.Empty".into() + "/rerun.sdk_comms.v0.WriteMessagesResponse".into() + } +} +#[derive(Clone, Copy, PartialEq, ::prost::Message)] +pub struct ReadMessagesRequest {} +impl ::prost::Name for ReadMessagesRequest { + const NAME: &'static str = "ReadMessagesRequest"; + const PACKAGE: &'static str = "rerun.sdk_comms.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.sdk_comms.v0.ReadMessagesRequest".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.sdk_comms.v0.ReadMessagesRequest".into() } } /// Generated client implementations. @@ -106,7 +118,8 @@ pub mod message_proxy_client { request: impl tonic::IntoStreamingRequest< Message = super::super::super::log_msg::v0::LogMsg, >, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result, tonic::Status> + { self.inner.ready().await.map_err(|e| { tonic::Status::unknown(format!("Service was not ready: {}", e.into())) })?; @@ -123,7 +136,7 @@ pub mod message_proxy_client { } pub async fn read_messages( &mut self, - request: impl tonic::IntoRequest, + request: impl tonic::IntoRequest, ) -> std::result::Result< tonic::Response>, tonic::Status, @@ -162,7 +175,7 @@ pub mod message_proxy_server { async fn write_messages( &self, request: tonic::Request>, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result, tonic::Status>; /// Server streaming response type for the ReadMessages method. type ReadMessagesStream: tonic::codegen::tokio_stream::Stream< Item = std::result::Result, @@ -170,7 +183,7 @@ pub mod message_proxy_server { + 'static; async fn read_messages( &self, - request: tonic::Request, + request: tonic::Request, ) -> std::result::Result, tonic::Status>; } /// Simple buffer for messages between SDKs and viewers. @@ -263,7 +276,7 @@ pub mod message_proxy_server { super::super::super::log_msg::v0::LogMsg, > for WriteMessagesSvc { - type Response = super::Empty; + type Response = super::WriteMessagesResponse; type Future = BoxFuture, tonic::Status>; fn call( &mut self, @@ -303,12 +316,18 @@ pub mod message_proxy_server { "/rerun.sdk_comms.v0.MessageProxy/ReadMessages" => { #[allow(non_camel_case_types)] struct ReadMessagesSvc(pub Arc); - impl tonic::server::ServerStreamingService for ReadMessagesSvc { + impl + tonic::server::ServerStreamingService + for ReadMessagesSvc + { type Response = super::super::super::log_msg::v0::LogMsg; type ResponseStream = T::ReadMessagesStream; type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); let fut = async move { ::read_messages(&inner, request).await From 7ff0cd53c7c05ae8c2e264fc6a52007aa16496a9 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Mon, 10 Feb 2025 18:48:09 +0100 Subject: [PATCH 75/87] rename signal -> server_shutdown_signal --- crates/top/re_sdk/src/web_viewer.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/top/re_sdk/src/web_viewer.rs b/crates/top/re_sdk/src/web_viewer.rs index 75c02a1cb264..72b97eb36883 100644 --- a/crates/top/re_sdk/src/web_viewer.rs +++ b/crates/top/re_sdk/src/web_viewer.rs @@ -28,7 +28,7 @@ struct WebViewerSink { _server_handle: std::thread::JoinHandle<()>, /// Rerun websocket server. - server_signal: re_grpc_server::shutdown::Signal, + server_shutdown_signal: re_grpc_server::shutdown::Signal, /// The http server serving wasm & html. _webviewer_server: WebViewerServer, @@ -43,7 +43,7 @@ impl WebViewerSink { grpc_port: u16, server_memory_limit: re_memory::MemoryLimit, ) -> Result { - let (signal, shutdown) = re_grpc_server::shutdown::shutdown(); + let (server_shutdown_signal, shutdown) = re_grpc_server::shutdown::shutdown(); let grpc_server_addr = format!("{bind_ip}:{grpc_port}").parse()?; let (channel_tx, channel_rx) = re_smart_channel::smart_channel::( @@ -87,7 +87,7 @@ impl WebViewerSink { open_browser, sender: channel_tx, _server_handle: server_handle, - server_signal: signal, + server_shutdown_signal, _webviewer_server: webviewer_server, }) } @@ -118,7 +118,7 @@ impl Drop for WebViewerSink { std::thread::sleep(std::time::Duration::from_millis(1000)); } - self.server_signal.stop(); + self.server_shutdown_signal.stop(); } } From ac8e93b0ee002e6cbd5c7af29784ec19ee658409 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Mon, 10 Feb 2025 18:49:01 +0100 Subject: [PATCH 76/87] update comment --- crates/top/re_sdk/src/log_sink.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/top/re_sdk/src/log_sink.rs b/crates/top/re_sdk/src/log_sink.rs index 0e2874c16c74..1a00d8000e70 100644 --- a/crates/top/re_sdk/src/log_sink.rs +++ b/crates/top/re_sdk/src/log_sink.rs @@ -333,7 +333,7 @@ impl LogSink for CallbackSink { // ---------------------------------------------------------------------------- -/// Stream log messages to an in-memory storage node. +/// Stream log messages to an a remote Rerun server. pub struct GrpcSink { client: MessageProxyClient, } From 1bd3ecae91228036863e8dc097f21f746b9f31b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Proch=C3=A1zka?= Date: Mon, 10 Feb 2025 18:51:50 +0100 Subject: [PATCH 77/87] Update crates/top/re_sdk/src/lib.rs Co-authored-by: Jeremy Leibs --- crates/top/re_sdk/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/top/re_sdk/src/lib.rs b/crates/top/re_sdk/src/lib.rs index 449a61011e24..d76f1d25a822 100644 --- a/crates/top/re_sdk/src/lib.rs +++ b/crates/top/re_sdk/src/lib.rs @@ -33,7 +33,7 @@ pub use self::recording_stream::{ RecordingStreamResult, }; -/// The default potr of a Rerun gRPC server. +/// The default port of a Rerun gRPC server. pub const DEFAULT_SERVER_PORT: u16 = 9876; /// The default URL of a Rerun gRPC server. From c6c44a13d85e5f97779c25535d6e04116220f2c1 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Mon, 10 Feb 2025 21:09:53 +0100 Subject: [PATCH 78/87] improve docs for re_grpc_server --- crates/store/re_grpc_server/src/lib.rs | 41 +++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/crates/store/re_grpc_server/src/lib.rs b/crates/store/re_grpc_server/src/lib.rs index 4be3ba84c756..7300011ae341 100644 --- a/crates/store/re_grpc_server/src/lib.rs +++ b/crates/store/re_grpc_server/src/lib.rs @@ -27,9 +27,22 @@ use tower_http::cors::CorsLayer; pub const DEFAULT_SERVER_PORT: u16 = 9876; pub const DEFAULT_MEMORY_LIMIT: MemoryLimit = MemoryLimit::UNLIMITED; -/// Listen for incoming clients on `addr`. +/// Start a Rerun server, listening on `addr`. /// -/// The server runs on the current task. +/// A Rerun server is an in-memory implementation of a Storage Node. +/// +/// The returned future must be polled for the server to make progress. +/// +/// Currently, the only RPCs supported by the server are `WriteMessages` and `ReadMessages`. +/// +/// Clients send data to the server via `WriteMessages`. Any sent messages will be stored +/// in the server's message queue. Messages are only removed if the server hits its configured +/// memory limit. +/// +/// Clients receive data from the server via `ReadMessages`. Upon establishing the stream, +/// the server sends all messages stored in its message queue, and subscribes the client +/// to the queue. Any messages sent to the server through `WriteMessages` will be proxied +/// to the open `ReadMessages` stream. pub async fn serve( addr: SocketAddr, memory_limit: MemoryLimit, @@ -72,7 +85,17 @@ async fn serve_impl( .await } -pub async fn serve_with_send( +/// Start a Rerun server, listening on `addr`. +/// +/// The returned future must be polled for the server to make progress. +/// +/// This function additionally accepts a smart channel, through which messages +/// can be sent to the server directly. It is similar to creating a client +/// and sending messages through `WriteMessages`, but without the overhead +/// of a localhost connection. +/// +/// See [`serve`] for more information about what a Rerun server is. +pub async fn serve_from_channel( addr: SocketAddr, memory_limit: MemoryLimit, shutdown: shutdown::Shutdown, @@ -135,7 +158,17 @@ pub async fn serve_with_send( } } -/// Spawn the server and subscribe to its message queue. +/// Start a Rerun server, listening on `addr`. +/// +/// This function additionally creates a smart channel, and returns its receiving end. +/// Any messages received by the server are sent through the channel. This is similar +/// to creating a client and calling `ReadMessages`, but without the overhead of a +/// localhost connection. +/// +/// The server is spawned as a task on a `tokio` runtime. This function panics if the +/// runtime is not available. +/// +/// See [`serve`] for more information about what a Rerun server is. pub fn spawn_with_recv( addr: SocketAddr, memory_limit: MemoryLimit, From 47d856d2a32895366abcfa2011d19aa89b9fe9e5 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Mon, 10 Feb 2025 21:14:08 +0100 Subject: [PATCH 79/87] fix after rename --- crates/top/re_sdk/src/web_viewer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/top/re_sdk/src/web_viewer.rs b/crates/top/re_sdk/src/web_viewer.rs index 72b97eb36883..83179db0ebf6 100644 --- a/crates/top/re_sdk/src/web_viewer.rs +++ b/crates/top/re_sdk/src/web_viewer.rs @@ -59,7 +59,7 @@ impl WebViewerSink { builder.enable_all(); let rt = builder.build().expect("failed to build tokio runtime"); - rt.block_on(re_grpc_server::serve_with_send( + rt.block_on(re_grpc_server::serve_from_channel( grpc_server_addr, server_memory_limit, shutdown, From 94e0ad59ad98bfde7fcf9cdb3028daf774b2cb2d Mon Sep 17 00:00:00 2001 From: jprochazk Date: Mon, 10 Feb 2025 21:20:53 +0100 Subject: [PATCH 80/87] use const_format for default connect url --- Cargo.lock | 21 +++++++++++++++++++++ Cargo.toml | 1 + crates/top/re_sdk/Cargo.toml | 1 + crates/top/re_sdk/src/lib.rs | 3 ++- 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 476a6b810a2f..3e3e67c2dec1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1503,6 +1503,26 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "const_format" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "126f97965c8ad46d6d9163268ff28432e8f6a1196a55578867832e3049df63dd" +dependencies = [ + "const_format_proc_macros", +] + +[[package]] +name = "const_format_proc_macros" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + [[package]] name = "const_soft_float" version = "0.1.4" @@ -6478,6 +6498,7 @@ name = "re_sdk" version = "0.23.0-alpha.1+dev" dependencies = [ "ahash", + "const_format", "crossbeam", "document-features", "itertools 0.13.0", diff --git a/Cargo.toml b/Cargo.toml index 26a6210e64c9..3044127aec5c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -176,6 +176,7 @@ clean-path = "0.2" colored = "2.1" comfy-table = { version = "7.0", default-features = false } console_error_panic_hook = "0.1.6" +const_format = "0.2" convert_case = "0.6" criterion = "0.5" crossbeam = "0.8" diff --git a/crates/top/re_sdk/Cargo.toml b/crates/top/re_sdk/Cargo.toml index 4024ef46f239..e3c3f5fb04a4 100644 --- a/crates/top/re_sdk/Cargo.toml +++ b/crates/top/re_sdk/Cargo.toml @@ -62,6 +62,7 @@ re_memory.workspace = true re_types_core.workspace = true ahash.workspace = true +const_format.workspace = true crossbeam.workspace = true document-features.workspace = true itertools.workspace = true diff --git a/crates/top/re_sdk/src/lib.rs b/crates/top/re_sdk/src/lib.rs index d76f1d25a822..269445df5f43 100644 --- a/crates/top/re_sdk/src/lib.rs +++ b/crates/top/re_sdk/src/lib.rs @@ -39,7 +39,8 @@ pub const DEFAULT_SERVER_PORT: u16 = 9876; /// The default URL of a Rerun gRPC server. /// /// This isn't used to _host_ the server, only to _connect_ to it. -pub const DEFAULT_CONNECT_URL: &str = "http://127.0.0.1:9876"; +pub const DEFAULT_CONNECT_URL: &str = + const_format::concatcp!("http://127.0.0.1:{DEFAULT_SERVER_PORT}"); /// The default address of a Rerun TCP server which an SDK connects to. #[deprecated(since = "0.22.0", note = "migrate to connect_grpc")] From e1bd084ba8c92a9cf64789ed1ef52509173136b4 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Mon, 10 Feb 2025 22:26:56 +0100 Subject: [PATCH 81/87] reintroduce flush_timeout --- .../re_grpc_client/src/message_proxy/write.rs | 36 ++++-- crates/top/re_sdk/src/lib.rs | 1 - crates/top/re_sdk/src/log_sink.rs | 15 ++- crates/top/re_sdk/src/recording_stream.rs | 115 +++++++++++------- crates/top/rerun/src/clap.rs | 3 +- crates/top/rerun/src/commands/entrypoint.rs | 2 +- crates/top/rerun_c/src/lib.rs | 27 ++-- examples/rust/custom_callback/src/app.rs | 2 +- rerun_cpp/src/rerun/c/rerun.h | 4 +- rerun_cpp/src/rerun/recording_stream.cpp | 8 +- rerun_cpp/src/rerun/recording_stream.hpp | 14 ++- rerun_py/src/python_bridge.rs | 22 ++-- .../rust/test_data_density_graph/src/main.rs | 25 ++-- 13 files changed, 182 insertions(+), 92 deletions(-) diff --git a/crates/store/re_grpc_client/src/message_proxy/write.rs b/crates/store/re_grpc_client/src/message_proxy/write.rs index cbd0eb731d19..79c8836499f9 100644 --- a/crates/store/re_grpc_client/src/message_proxy/write.rs +++ b/crates/store/re_grpc_client/src/message_proxy/write.rs @@ -23,13 +23,15 @@ enum Cmd { #[derive(Clone)] pub struct Options { - compression: Compression, + pub compression: Compression, + pub flush_timeout: Option, } impl Default for Options { fn default() -> Self { Self { compression: Compression::LZ4, + flush_timeout: Default::default(), } } } @@ -38,6 +40,7 @@ pub struct Client { thread: Option>, cmd_tx: UnboundedSender, shutdown_tx: Sender<()>, + flush_timeout: Option, } impl Client { @@ -67,6 +70,7 @@ impl Client { thread: Some(thread), cmd_tx, shutdown_tx, + flush_timeout: options.flush_timeout, } } @@ -75,18 +79,34 @@ impl Client { } pub fn flush(&self) { - let (tx, rx) = oneshot::channel(); + use tokio::sync::oneshot::error::TryRecvError; + + let (tx, mut rx) = oneshot::channel(); if self.cmd_tx.send(Cmd::Flush(tx)).is_err() { re_log::debug!("Flush failed: already shut down."); return; }; - match rx.blocking_recv() { - Ok(_) => { - re_log::debug!("Flush complete"); - } - Err(_) => { - re_log::debug!("Flush failed, not all messages were sent"); + let now = std::time::Instant::now(); + loop { + match rx.try_recv() { + Ok(_) => { + re_log::debug!("Flush complete"); + } + Err(TryRecvError::Empty) => { + let Some(timeout) = self.flush_timeout else { + std::thread::yield_now(); + continue; + }; + + if now.elapsed() > timeout { + re_log::debug!("Flush timed out, not all messages were sent"); + return; + } + } + Err(TryRecvError::Closed) => { + re_log::debug!("Flush failed, not all messages were sent"); + } } } } diff --git a/crates/top/re_sdk/src/lib.rs b/crates/top/re_sdk/src/lib.rs index 269445df5f43..1d5ab5e68c3e 100644 --- a/crates/top/re_sdk/src/lib.rs +++ b/crates/top/re_sdk/src/lib.rs @@ -50,7 +50,6 @@ pub fn default_server_addr() -> std::net::SocketAddr { /// The default amount of time to wait for the TCP connection to resume during a flush #[allow(clippy::unnecessary_wraps)] -#[deprecated(since = "0.22.0", note = "flush timeout no longer has any effect")] pub fn default_flush_timeout() -> Option { // NOTE: This is part of the SDK and meant to be used where we accept `Option` values. Some(std::time::Duration::from_secs(2)) diff --git a/crates/top/re_sdk/src/log_sink.rs b/crates/top/re_sdk/src/log_sink.rs index 1a00d8000e70..47a78fcc05db 100644 --- a/crates/top/re_sdk/src/log_sink.rs +++ b/crates/top/re_sdk/src/log_sink.rs @@ -1,8 +1,9 @@ use std::fmt; use std::sync::Arc; +use std::time::Duration; use parking_lot::Mutex; -use re_grpc_client::message_proxy::write::Client as MessageProxyClient; +use re_grpc_client::message_proxy::write::{Client as MessageProxyClient, Options}; use re_grpc_client::message_proxy::MessageProxyUrl; use re_log_encoding::encoder::encode_as_bytes_local; use re_log_encoding::encoder::{local_raw_encoder, EncodeError}; @@ -341,15 +342,23 @@ pub struct GrpcSink { impl GrpcSink { /// Connect to the in-memory storage node over HTTP. /// + /// `flush_timeout` is the minimum time the [`TcpSink`] will wait during a flush + /// before potentially dropping data. Note: Passing `None` here can cause a + /// call to `flush` to block indefinitely if a connection cannot be established. + /// /// ### Example /// /// ```ignore /// GrpcSink::new("http://127.0.0.1:9434"); /// ``` #[inline] - pub fn new(url: MessageProxyUrl) -> Self { + pub fn new(url: MessageProxyUrl, flush_timeout: Option) -> Self { + let options = Options { + flush_timeout, + ..Default::default() + }; Self { - client: MessageProxyClient::new(url, Default::default()), + client: MessageProxyClient::new(url, options), } } } diff --git a/crates/top/re_sdk/src/recording_stream.rs b/crates/top/re_sdk/src/recording_stream.rs index b84c977ce77c..e6f5c203e524 100644 --- a/crates/top/re_sdk/src/recording_stream.rs +++ b/crates/top/re_sdk/src/recording_stream.rs @@ -2,6 +2,7 @@ use std::fmt; use std::io::IsTerminal; use std::sync::Weak; use std::sync::{atomic::AtomicI64, Arc}; +use std::time::Duration; use ahash::HashMap; use crossbeam::channel::{Receiver, Sender}; @@ -336,6 +337,10 @@ impl RecordingStreamBuilder { /// Creates a new [`RecordingStream`] that is pre-configured to stream the data through to a /// remote Rerun instance. /// + /// `flush_timeout` is the minimum time the [`GrpcSink`][`crate::log_sink::GrpcSink`] will + /// wait during a flush before potentially dropping data. Note: Passing `None` here can cause a + /// call to `flush` to block indefinitely if a connection cannot be established. + /// /// ## Example /// /// ```no_run @@ -347,15 +352,19 @@ impl RecordingStreamBuilder { pub fn connect_opts( self, addr: std::net::SocketAddr, - flush_timeout: Option, + flush_timeout: Option, ) -> RecordingStreamResult { let _ = flush_timeout; - self.connect_grpc_opts(format!("http://{addr}")) + self.connect_grpc_opts(format!("http://{addr}"), flush_timeout) } /// Creates a new [`RecordingStream`] that is pre-configured to stream the data through to a /// remote Rerun instance. /// + /// `flush_timeout` is the minimum time the [`GrpcSink`][`crate::log_sink::GrpcSink`] will + /// wait during a flush before potentially dropping data. Note: Passing `None` here can cause a + /// call to `flush` to block indefinitely if a connection cannot be established. + /// /// ## Example /// /// ```no_run @@ -367,10 +376,9 @@ impl RecordingStreamBuilder { pub fn connect_tcp_opts( self, addr: std::net::SocketAddr, - flush_timeout: Option, + flush_timeout: Option, ) -> RecordingStreamResult { - let _ = flush_timeout; - self.connect_grpc_opts(format!("http://{addr}")) + self.connect_grpc_opts(format!("http://{addr}"), flush_timeout) } /// Creates a new [`RecordingStream`] that is pre-configured to stream the data through to a @@ -385,15 +393,19 @@ impl RecordingStreamBuilder { /// # Ok::<(), Box>(()) /// ``` pub fn connect_grpc(self) -> RecordingStreamResult { - self.connect_grpc_opts(format!( - "http://127.0.0.1:{}", - re_grpc_server::DEFAULT_SERVER_PORT - )) + self.connect_grpc_opts( + format!("http://127.0.0.1:{}", re_grpc_server::DEFAULT_SERVER_PORT), + crate::default_flush_timeout(), + ) } /// Creates a new [`RecordingStream`] that is pre-configured to stream the data through to a /// remote Rerun instance. /// + /// `flush_timeout` is the minimum time the [`GrpcSink`][`crate::log_sink::GrpcSink`] will + /// wait during a flush before potentially dropping data. Note: Passing `None` here can cause a + /// call to `flush` to block indefinitely if a connection cannot be established. + /// /// ## Example /// /// ```no_run @@ -404,13 +416,17 @@ impl RecordingStreamBuilder { pub fn connect_grpc_opts( self, url: impl Into, + flush_timeout: Option, ) -> RecordingStreamResult { let (enabled, store_info, batcher_config) = self.into_args(); if enabled { RecordingStream::new( store_info, batcher_config, - Box::new(crate::log_sink::GrpcSink::new(url.into().parse()?)), + Box::new(crate::log_sink::GrpcSink::new( + url.into().parse()?, + flush_timeout, + )), ) } else { re_log::debug!("Rerun disabled - call to connect() ignored"); @@ -498,7 +514,7 @@ impl RecordingStreamBuilder { /// # Ok::<(), Box>(()) /// ``` pub fn spawn(self) -> RecordingStreamResult { - self.spawn_opts(&Default::default()) + self.spawn_opts(&Default::default(), crate::default_flush_timeout()) } /// Spawns a new Rerun Viewer process from an executable available in PATH, then creates a new @@ -510,14 +526,22 @@ impl RecordingStreamBuilder { /// The behavior of the spawned Viewer can be configured via `opts`. /// If you're fine with the default behavior, refer to the simpler [`Self::spawn`]. /// + /// `flush_timeout` is the minimum time the [`GrpcSink`][`crate::log_sink::GrpcSink`] will + /// wait during a flush before potentially dropping data. Note: Passing `None` here can cause a + /// call to `flush` to block indefinitely if a connection cannot be established. + /// /// ## Example /// /// ```no_run /// let rec = re_sdk::RecordingStreamBuilder::new("rerun_example_app") - /// .spawn_opts(&re_sdk::SpawnOptions::default())?; + /// .spawn_opts(&re_sdk::SpawnOptions::default(), re_sdk::default_flush_timeout())?; /// # Ok::<(), Box>(()) /// ``` - pub fn spawn_opts(self, opts: &crate::SpawnOptions) -> RecordingStreamResult { + pub fn spawn_opts( + self, + opts: &crate::SpawnOptions, + flush_timeout: Option, + ) -> RecordingStreamResult { if !self.is_enabled() { re_log::debug!("Rerun disabled - call to spawn() ignored"); return Ok(RecordingStream::disabled()); @@ -528,12 +552,12 @@ impl RecordingStreamBuilder { // NOTE: If `_RERUN_TEST_FORCE_SAVE` is set, all recording streams will write to disk no matter // what, thus spawning a viewer is pointless (and probably not intended). if forced_sink_path().is_some() { - return self.connect_grpc_opts(url); + return self.connect_grpc_opts(url, flush_timeout); } crate::spawn(opts)?; - self.connect_grpc_opts(url) + self.connect_grpc_opts(url, flush_timeout) } /// Creates a new [`RecordingStream`] that is pre-configured to stream the data through to a @@ -1715,7 +1739,7 @@ impl RecordingStream { /// See [`Self::set_sink`] for more information. #[deprecated(since = "0.22.0", note = "use connect_grpc() instead")] pub fn connect(&self) { - self.connect_grpc(); + self.connect_grpc().expect("failed to connect via gRPC"); } /// Swaps the underlying sink for a sink pre-configured to use the specified address. @@ -1723,18 +1747,15 @@ impl RecordingStream { /// This is a convenience wrapper for [`Self::set_sink`] that upholds the same guarantees in /// terms of data durability and ordering. /// See [`Self::set_sink`] for more information. + /// + /// `flush_timeout` is the minimum time the [`GrpcSink`][`crate::log_sink::GrpcSink`] will + /// wait during a flush before potentially dropping data. Note: Passing `None` here can cause a + /// call to `flush` to block indefinitely if a connection cannot be established. #[deprecated(since = "0.22.0", note = "use connect_grpc() instead")] - pub fn connect_opts( - &self, - addr: std::net::SocketAddr, - flush_timeout: Option, - ) { + pub fn connect_opts(&self, addr: std::net::SocketAddr, flush_timeout: Option) { let _ = flush_timeout; - self.connect_grpc_opts( - format!("http://{addr}") - .parse() - .expect("should always be valid"), - ); + self.connect_grpc_opts(format!("http://{addr}"), flush_timeout) + .expect("failed to connect via gRPC"); } /// Swaps the underlying sink for a [`crate::log_sink::GrpcSink`] sink pre-configured to use @@ -1745,12 +1766,11 @@ impl RecordingStream { /// This is a convenience wrapper for [`Self::set_sink`] that upholds the same guarantees in /// terms of data durability and ordering. /// See [`Self::set_sink`] for more information. - pub fn connect_grpc(&self) { + pub fn connect_grpc(&self) -> RecordingStreamResult<()> { self.connect_grpc_opts( - format!("http://127.0.0.1:{}", re_grpc_server::DEFAULT_SERVER_PORT) - .parse() - .expect("should always be valid"), - ); + format!("http://127.0.0.1:{}", re_grpc_server::DEFAULT_SERVER_PORT), + crate::default_flush_timeout(), + ) } /// Swaps the underlying sink for a [`crate::log_sink::GrpcSink`] sink pre-configured to use @@ -1759,15 +1779,24 @@ impl RecordingStream { /// This is a convenience wrapper for [`Self::set_sink`] that upholds the same guarantees in /// terms of data durability and ordering. /// See [`Self::set_sink`] for more information. - pub fn connect_grpc_opts(&self, url: re_grpc_client::MessageProxyUrl) { + /// + /// `flush_timeout` is the minimum time the [`GrpcSink`][`crate::log_sink::GrpcSink`] will + /// wait during a flush before potentially dropping data. Note: Passing `None` here can cause a + /// call to `flush` to block indefinitely if a connection cannot be established. + pub fn connect_grpc_opts( + &self, + url: impl Into, + flush_timeout: Option, + ) -> RecordingStreamResult<()> { if forced_sink_path().is_some() { re_log::debug!("Ignored setting new GrpcSink since {ENV_FORCE_SAVE} is set"); - return; + return Ok(()); } - let sink = crate::log_sink::GrpcSink::new(url); + let sink = crate::log_sink::GrpcSink::new(url.into().parse()?, flush_timeout); self.set_sink(Box::new(sink)); + Ok(()) } /// Spawns a new Rerun Viewer process from an executable available in PATH, then swaps the @@ -1784,7 +1813,7 @@ impl RecordingStream { /// terms of data durability and ordering. /// See [`Self::set_sink`] for more information. pub fn spawn(&self) -> RecordingStreamResult<()> { - self.spawn_opts(&Default::default()) + self.spawn_opts(&Default::default(), crate::default_flush_timeout()) } /// Spawns a new Rerun Viewer process from an executable available in PATH, then swaps the @@ -1800,7 +1829,15 @@ impl RecordingStream { /// This is a convenience wrapper for [`Self::set_sink`] that upholds the same guarantees in /// terms of data durability and ordering. /// See [`Self::set_sink`] for more information. - pub fn spawn_opts(&self, opts: &crate::SpawnOptions) -> RecordingStreamResult<()> { + /// + /// `flush_timeout` is the minimum time the [`GrpcSink`][`crate::log_sink::GrpcSink`] will + /// wait during a flush before potentially dropping data. Note: Passing `None` here can cause a + /// call to `flush` to block indefinitely if a connection cannot be established. + pub fn spawn_opts( + &self, + opts: &crate::SpawnOptions, + flush_timeout: Option, + ) -> RecordingStreamResult<()> { if !self.is_enabled() { re_log::debug!("Rerun disabled - call to spawn() ignored"); return Ok(()); @@ -1812,11 +1849,7 @@ impl RecordingStream { crate::spawn(opts)?; - self.connect_grpc_opts( - format!("http://{}", opts.connect_addr()) - .parse() - .expect("should always be valid"), - ); + self.connect_grpc_opts(format!("http://{}", opts.connect_addr()), flush_timeout)?; Ok(()) } diff --git a/crates/top/rerun/src/clap.rs b/crates/top/rerun/src/clap.rs index d8a9966df2e6..4e16b9829846 100644 --- a/crates/top/rerun/src/clap.rs +++ b/crates/top/rerun/src/clap.rs @@ -111,7 +111,8 @@ impl RerunArgs { )), RerunBehavior::Connect(url) => Ok(( - RecordingStreamBuilder::new(application_id).connect_grpc_opts(url)?, + RecordingStreamBuilder::new(application_id) + .connect_grpc_opts(url, re_sdk::default_flush_timeout())?, Default::default(), )), diff --git a/crates/top/rerun/src/commands/entrypoint.rs b/crates/top/rerun/src/commands/entrypoint.rs index 775a0b52c9c1..dfb5dab7f9c8 100644 --- a/crates/top/rerun/src/commands/entrypoint.rs +++ b/crates/top/rerun/src/commands/entrypoint.rs @@ -807,7 +807,7 @@ fn run_impl( // This spawns its own single-threaded runtime on a separate thread, // no need to `rt.enter()`: - let sink = re_sdk::sink::GrpcSink::new(url); + let sink = re_sdk::sink::GrpcSink::new(url, crate::default_flush_timeout()); for rx in rxs { while rx.is_connected() { diff --git a/crates/top/rerun_c/src/lib.rs b/crates/top/rerun_c/src/lib.rs index 2ea4ffa754ae..6b50990dd01f 100644 --- a/crates/top/rerun_c/src/lib.rs +++ b/crates/top/rerun_c/src/lib.rs @@ -602,14 +602,19 @@ pub extern "C" fn rr_recording_stream_connect( fn rr_recording_stream_connect_grpc_impl( stream: CRecordingStream, url: CStringView, + flush_timeout_sec: f32, ) -> Result<(), CError> { let stream = recording_stream(stream)?; - let url = url.as_str("url")?.parse(); + let url = url.as_str("url")?; + let flush_timeout = if flush_timeout_sec >= 0.0 { + Some(std::time::Duration::from_secs_f32(flush_timeout_sec)) + } else { + None + }; - match url { - Ok(url) => stream.connect_grpc_opts(url), - Err(err) => return Err(CError::new(CErrorCode::InvalidServerUrl, &err.to_string())), + if let Err(err) = stream.connect_grpc_opts(url, flush_timeout) { + return Err(CError::new(CErrorCode::InvalidServerUrl, &err.to_string())); } Ok(()) @@ -620,9 +625,10 @@ fn rr_recording_stream_connect_grpc_impl( pub extern "C" fn rr_recording_stream_connect_grpc( id: CRecordingStream, url: CStringView, + flush_timeout_sec: f32, error: *mut CError, ) { - if let Err(err) = rr_recording_stream_connect_grpc_impl(id, url) { + if let Err(err) = rr_recording_stream_connect_grpc_impl(id, url, flush_timeout_sec) { err.write_error(error); } } @@ -631,6 +637,7 @@ pub extern "C" fn rr_recording_stream_connect_grpc( fn rr_recording_stream_spawn_impl( stream: CRecordingStream, spawn_opts: *const CSpawnOptions, + flush_timeout_sec: f32, ) -> Result<(), CError> { let stream = recording_stream(stream)?; @@ -640,9 +647,14 @@ fn rr_recording_stream_spawn_impl( let spawn_opts = ptr::try_ptr_as_ref(spawn_opts, "spawn_opts")?; spawn_opts.as_rust()? }; + let flush_timeout = if flush_timeout_sec >= 0.0 { + Some(std::time::Duration::from_secs_f32(flush_timeout_sec)) + } else { + None + }; stream - .spawn_opts(&spawn_opts) + .spawn_opts(&spawn_opts, flush_timeout) .map_err(|err| CError::new(CErrorCode::RecordingStreamSpawnFailure, &err.to_string()))?; Ok(()) @@ -653,9 +665,10 @@ fn rr_recording_stream_spawn_impl( pub extern "C" fn rr_recording_stream_spawn( id: CRecordingStream, spawn_opts: *const CSpawnOptions, + flush_timeout_sec: f32, error: *mut CError, ) { - if let Err(err) = rr_recording_stream_spawn_impl(id, spawn_opts) { + if let Err(err) = rr_recording_stream_spawn_impl(id, spawn_opts, flush_timeout_sec) { err.write_error(error); } } diff --git a/examples/rust/custom_callback/src/app.rs b/examples/rust/custom_callback/src/app.rs index 4af116efc08b..8254dddafb6a 100644 --- a/examples/rust/custom_callback/src/app.rs +++ b/examples/rust/custom_callback/src/app.rs @@ -15,7 +15,7 @@ use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; async fn main() -> Result<(), Box> { let mut app = ControlApp::bind("127.0.0.1:8888").await?.run(); let rec = rerun::RecordingStreamBuilder::new("rerun_example_custom_callback") - .connect_grpc_opts("http://127.0.0.1:9877")?; + .connect_grpc_opts("http://127.0.0.1:9877", rerun::default_flush_timeout())?; // Add a handler for incoming messages let add_rec = rec.clone(); diff --git a/rerun_cpp/src/rerun/c/rerun.h b/rerun_cpp/src/rerun/c/rerun.h index 6b4b3adf3f71..1d097fe56a89 100644 --- a/rerun_cpp/src/rerun/c/rerun.h +++ b/rerun_cpp/src/rerun/c/rerun.h @@ -441,7 +441,7 @@ extern void rr_recording_stream_connect( /// This function returns immediately and will only raise an error for argument parsing errors, /// not for connection errors as these happen asynchronously. extern void rr_recording_stream_connect_grpc( - rr_recording_stream stream, rr_string url, rr_error* error + rr_recording_stream stream, rr_string url, float flush_timeout_sec, rr_error* error ); /// Spawns a new Rerun Viewer process from an executable available in PATH, then connects to it @@ -462,7 +462,7 @@ extern void rr_recording_stream_connect_grpc( /// dropping data if progress is not being made. Passing a negative value indicates no timeout, /// and can cause a call to `flush` to block indefinitely. extern void rr_recording_stream_spawn( - rr_recording_stream stream, const rr_spawn_options* spawn_opts, rr_error* error + rr_recording_stream stream, const rr_spawn_options* spawn_opts, float flush_timeout_sec, rr_error* error ); /// Stream all log-data to a given `.rrd` file. diff --git a/rerun_cpp/src/rerun/recording_stream.cpp b/rerun_cpp/src/rerun/recording_stream.cpp index 19d4d43b0332..35f33a64d757 100644 --- a/rerun_cpp/src/rerun/recording_stream.cpp +++ b/rerun_cpp/src/rerun/recording_stream.cpp @@ -127,17 +127,17 @@ namespace rerun { return status; } - Error RecordingStream::connect_grpc(std::string_view url) const { + Error RecordingStream::connect_grpc(std::string_view url, float flush_timeout_sec) const { rr_error status = {}; - rr_recording_stream_connect_grpc(_id, detail::to_rr_string(url), &status); + rr_recording_stream_connect_grpc(_id, detail::to_rr_string(url), flush_timeout_sec, &status); return status; } - Error RecordingStream::spawn(const SpawnOptions& options) const { + Error RecordingStream::spawn(const SpawnOptions& options, float flush_timeout_sec) const { rr_spawn_options rerun_c_options = {}; options.fill_rerun_c_struct(rerun_c_options); rr_error status = {}; - rr_recording_stream_spawn(_id, &rerun_c_options, &status); + rr_recording_stream_spawn(_id, &rerun_c_options, flush_timeout_sec, &status); return status; } diff --git a/rerun_cpp/src/rerun/recording_stream.hpp b/rerun_cpp/src/rerun/recording_stream.hpp index c2750d0aa2de..22ee016c84d7 100644 --- a/rerun_cpp/src/rerun/recording_stream.hpp +++ b/rerun_cpp/src/rerun/recording_stream.hpp @@ -165,19 +165,29 @@ namespace rerun { /// /// Requires that you first start a Rerun Viewer by typing 'rerun' in a terminal. /// + /// flush_timeout_sec: + /// The minimum time the SDK will wait during a flush before potentially + /// dropping data if progress is not being made. Passing a negative value indicates no + /// timeout, and can cause a call to `flush` to block indefinitely. + /// /// This function returns immediately. - Error connect_grpc(std::string_view url = "http://127.0.0.1:9876") const; + Error connect_grpc(std::string_view url = "http://127.0.0.1:9876", float flush_timeout_sec = 2.0) const; /// Spawns a new Rerun Viewer process from an executable available in PATH, then connects to it /// over gRPC. /// + /// flush_timeout_sec: + /// The minimum time the SDK will wait during a flush before potentially + /// dropping data if progress is not being made. Passing a negative value indicates no + /// timeout, and can cause a call to `flush` to block indefinitely. + /// /// If a Rerun Viewer is already listening on this port, the stream will be redirected to /// that viewer instead of starting a new one. /// /// ## Parameters /// options: /// See `rerun::SpawnOptions` for more information. - Error spawn(const SpawnOptions& options = {}) const; + Error spawn(const SpawnOptions& options = {}, float flush_timeout_sec = 2.0) const; /// @see RecordingStream::spawn template diff --git a/rerun_py/src/python_bridge.rs b/rerun_py/src/python_bridge.rs index c0ac940e4936..f69e8de4634a 100644 --- a/rerun_py/src/python_bridge.rs +++ b/rerun_py/src/python_bridge.rs @@ -585,9 +585,10 @@ fn spawn( } #[pyfunction] -#[pyo3(signature = (url, default_blueprint = None, recording = None))] +#[pyo3(signature = (url, flush_timeout_sec=re_sdk::default_flush_timeout().expect("always Some()").as_secs_f32(), default_blueprint = None, recording = None))] fn connect_grpc( url: Option, + flush_timeout_sec: Option, default_blueprint: Option<&PyMemorySinkStorage>, recording: Option<&PyRecordingStream>, py: Python<'_>, @@ -608,7 +609,10 @@ fn connect_grpc( } py.allow_threads(|| { - let sink = re_sdk::sink::GrpcSink::new(url); + let sink = re_sdk::sink::GrpcSink::new( + url, + flush_timeout_sec.map(std::time::Duration::from_secs_f32), + ); if let Some(default_blueprint) = default_blueprint { send_mem_sink_as_default_blueprint(&sink, default_blueprint); @@ -632,11 +636,7 @@ fn connect_grpc_blueprint( blueprint_stream: &PyRecordingStream, py: Python<'_>, ) -> PyResult<()> { - use re_sdk::external::re_grpc_server::DEFAULT_SERVER_PORT; - let url = url - .unwrap_or_else(|| format!("http://127.0.0.1:{DEFAULT_SERVER_PORT}")) - .parse::() - .map_err(|err| PyRuntimeError::new_err(err.to_string()))?; + let url = url.unwrap_or_else(|| re_sdk::DEFAULT_CONNECT_URL.to_owned()); if let Some(blueprint_id) = (*blueprint_stream).store_info().map(|info| info.store_id) { // The call to save, needs to flush. @@ -653,10 +653,12 @@ fn connect_grpc_blueprint( blueprint_stream.record_msg(activation_cmd.into()); - blueprint_stream.connect_grpc_opts(url); + blueprint_stream + .connect_grpc_opts(url, None) + .map_err(|err| PyRuntimeError::new_err(err.to_string()))?; flush_garbage_queue(); - }); - Ok(()) + Ok(()) + }) } else { Err(PyRuntimeError::new_err( "Blueprint stream has no store info".to_owned(), diff --git a/tests/rust/test_data_density_graph/src/main.rs b/tests/rust/test_data_density_graph/src/main.rs index e3a833fccb03..e962ff226492 100644 --- a/tests/rust/test_data_density_graph/src/main.rs +++ b/tests/rust/test_data_density_graph/src/main.rs @@ -14,18 +14,21 @@ fn main() -> anyhow::Result<()> { re_log::setup_logging(); let rec = rerun::RecordingStreamBuilder::new("rerun_example_test_data_density_graph") - .spawn_opts(&rerun::SpawnOptions { - wait_for_bind: true, - extra_env: { - use re_chunk_store::ChunkStoreConfig as C; - vec![ - (C::ENV_CHUNK_MAX_BYTES.into(), "0".into()), - (C::ENV_CHUNK_MAX_ROWS.into(), "0".into()), - (C::ENV_CHUNK_MAX_ROWS_IF_UNSORTED.into(), "0".into()), - ] + .spawn_opts( + &rerun::SpawnOptions { + wait_for_bind: true, + extra_env: { + use re_chunk_store::ChunkStoreConfig as C; + vec![ + (C::ENV_CHUNK_MAX_BYTES.into(), "0".into()), + (C::ENV_CHUNK_MAX_ROWS.into(), "0".into()), + (C::ENV_CHUNK_MAX_ROWS_IF_UNSORTED.into(), "0".into()), + ] + }, + ..Default::default() }, - ..Default::default() - })?; + rerun::default_flush_timeout(), + )?; run(&rec) } From 730943cd6121ac82d388eba6d07c159b770d5a66 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Mon, 10 Feb 2025 22:48:44 +0100 Subject: [PATCH 82/87] fix missing flush timeout in doctest --- crates/top/re_sdk/src/recording_stream.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/top/re_sdk/src/recording_stream.rs b/crates/top/re_sdk/src/recording_stream.rs index e6f5c203e524..16a9b499b6f6 100644 --- a/crates/top/re_sdk/src/recording_stream.rs +++ b/crates/top/re_sdk/src/recording_stream.rs @@ -410,7 +410,7 @@ impl RecordingStreamBuilder { /// /// ```no_run /// let rec = re_sdk::RecordingStreamBuilder::new("rerun_example_app") - /// .connect_grpc_opts("http://127.0.0.1:9876")?; + /// .connect_grpc_opts("http://127.0.0.1:9876", re_sdk::default_flush_timeout())?; /// # Ok::<(), Box>(()) /// ``` pub fn connect_grpc_opts( From e028513b65343cf499db96532ce86a741e3eaec4 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Mon, 10 Feb 2025 22:48:58 +0100 Subject: [PATCH 83/87] busy-wait for timeout in grpc sink flush --- .../re_grpc_client/src/message_proxy/write.rs | 45 +++---------------- 1 file changed, 7 insertions(+), 38 deletions(-) diff --git a/crates/store/re_grpc_client/src/message_proxy/write.rs b/crates/store/re_grpc_client/src/message_proxy/write.rs index 79c8836499f9..591882e15fa7 100644 --- a/crates/store/re_grpc_client/src/message_proxy/write.rs +++ b/crates/store/re_grpc_client/src/message_proxy/write.rs @@ -87,11 +87,12 @@ impl Client { return; }; - let now = std::time::Instant::now(); + let start = std::time::Instant::now(); loop { match rx.try_recv() { Ok(_) => { re_log::debug!("Flush complete"); + break; } Err(TryRecvError::Empty) => { let Some(timeout) = self.flush_timeout else { @@ -99,13 +100,15 @@ impl Client { continue; }; - if now.elapsed() > timeout { + let elapsed = start.elapsed(); + if elapsed >= timeout { re_log::debug!("Flush timed out, not all messages were sent"); - return; + break; } } Err(TryRecvError::Closed) => { re_log::debug!("Flush failed, not all messages were sent"); + break; } } } @@ -147,39 +150,17 @@ async fn message_proxy_client( } }; - // Temporarily buffer messages while we're connecting: - let mut buffered_messages = vec![]; let channel = loop { match endpoint.connect().await { Ok(channel) => break channel, Err(err) => { re_log::debug!("failed to connect to message proxy server: {err}"); tokio::select! { - cmd = cmd_rx.recv() => { - match cmd { - Some(Cmd::LogMsg(msg)) => { - buffered_messages.push(msg); - } - Some(Cmd::Flush(tx)) => { - re_log::warn_once!( - "Attempted to flush while gRPC client was connecting." - ); - if tx.send(()).is_err() { - re_log::debug!("Failed to respond to flush: channel is closed"); - return; - }; - } - None => { - re_log::debug!("Channel closed"); - return; - } - } - } _ = shutdown_rx.recv() => { re_log::debug!("shutting down client without flush"); return; } - _ = tokio::time::sleep(Duration::from_millis(200)) => { + _ = tokio::time::sleep(Duration::from_millis(100)) => { continue; } } @@ -189,18 +170,6 @@ async fn message_proxy_client( let mut client = MessageProxyClient::new(channel); let stream = async_stream::stream! { - for msg in buffered_messages { - let msg = match re_log_encoding::protobuf_conversions::log_msg_to_proto(msg, compression) { - Ok(msg) => msg, - Err(err) => { - re_log::error!("Failed to encode message: {err}"); - break; - } - }; - - yield msg; - } - loop { tokio::select! { cmd = cmd_rx.recv() => { From 7438f53af83692f5091e3d98dcee9ce22e60dce4 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Mon, 10 Feb 2025 22:49:12 +0100 Subject: [PATCH 84/87] lower flush timeout in cpp tests --- rerun_cpp/tests/recording_stream.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rerun_cpp/tests/recording_stream.cpp b/rerun_cpp/tests/recording_stream.cpp index 2dc8da419743..a4496f91b982 100644 --- a/rerun_cpp/tests/recording_stream.cpp +++ b/rerun_cpp/tests/recording_stream.cpp @@ -314,7 +314,7 @@ RR_DISABLE_DEPRECATION_WARNING // TODO(jan): Remove once `connect` is removed AND_GIVEN("an invalid address for the socket address") { THEN("connect call fails") { CHECK( - stream.connect("definitely not valid!").code == + stream.connect("definitely not valid!", 0.1f).code == rerun::ErrorCode::InvalidSocketAddress ); } @@ -322,7 +322,7 @@ RR_DISABLE_DEPRECATION_WARNING // TODO(jan): Remove once `connect` is removed AND_GIVEN("a valid socket address " << address) { THEN("connect call returns no error") { - CHECK(stream.connect(address).code == rerun::ErrorCode::Ok); + CHECK(stream.connect(address, 0.1f).code == rerun::ErrorCode::Ok); WHEN("logging an archetype and then flushing") { check_logged_error([&] { @@ -368,14 +368,14 @@ void test_logging_to_grpc_connection(const char* url, const rerun::RecordingStre AND_GIVEN("an invalid url") { THEN("connect call fails") { CHECK( - stream.connect_grpc("definitely not valid!").code == + stream.connect_grpc("definitely not valid!", 0.1f).code == rerun::ErrorCode::InvalidServerUrl ); } } AND_GIVEN("a valid socket url " << url) { THEN("connect call returns no error") { - CHECK(stream.connect_grpc(url).code == rerun::ErrorCode::Ok); + CHECK(stream.connect_grpc(url, 0.1f).code == rerun::ErrorCode::Ok); WHEN("logging an archetype and then flushing") { check_logged_error([&] { From 7cd49d5f8834c4e602e272411fc3b723e591a244 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Mon, 10 Feb 2025 22:56:00 +0100 Subject: [PATCH 85/87] reintroduce flush_timeout_sec for py connect apis --- rerun_py/rerun_sdk/rerun/sinks.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/rerun_py/rerun_sdk/rerun/sinks.py b/rerun_py/rerun_sdk/rerun/sinks.py index 7d0cb7a26b77..0160e16bd12d 100644 --- a/rerun_py/rerun_sdk/rerun/sinks.py +++ b/rerun_py/rerun_sdk/rerun/sinks.py @@ -42,7 +42,9 @@ def connect( addr: The ip:port to connect to flush_timeout_sec: - Deprecated. + The minimum time the SDK will wait during a flush before potentially + dropping data if progress is not being made. Passing `None` indicates no timeout, + and can cause a call to `flush` to block indefinitely. default_blueprint: Optionally set a default blueprint to use for this application. If the application already has an active blueprint, the new blueprint won't become active until the user @@ -58,6 +60,7 @@ def connect( addr = f"http://{addr}" return connect_grpc( url=addr, + flush_timeout_sec=flush_timeout_sec, default_blueprint=default_blueprint, recording=recording, # NOLINT: conversion not needed ) @@ -84,7 +87,9 @@ def connect_tcp( addr: The ip:port to connect to flush_timeout_sec: - Deprecated. + The minimum time the SDK will wait during a flush before potentially + dropping data if progress is not being made. Passing `None` indicates no timeout, + and can cause a call to `flush` to block indefinitely. default_blueprint: Optionally set a default blueprint to use for this application. If the application already has an active blueprint, the new blueprint won't become active until the user @@ -100,6 +105,7 @@ def connect_tcp( addr = f"http://{addr}" return connect_grpc( url=addr, + flush_timeout_sec=flush_timeout_sec, default_blueprint=default_blueprint, recording=recording, # NOLINT: conversion not needed ) @@ -108,6 +114,7 @@ def connect_tcp( def connect_grpc( url: str | None = None, *, + flush_timeout_sec: float | None = 2.0, default_blueprint: BlueprintLike | None = None, recording: RecordingStream | None = None, ) -> None: @@ -120,6 +127,10 @@ def connect_grpc( ---------- url: The HTTP(S) URL to connect to + flush_timeout_sec: + The minimum time the SDK will wait during a flush before potentially + dropping data if progress is not being made. Passing `None` indicates no timeout, + and can cause a call to `flush` to block indefinitely. default_blueprint Optionally set a default blueprint to use for this application. If the application already has an active blueprint, the new blueprint won't become active until the user @@ -150,6 +161,7 @@ def connect_grpc( bindings.connect_grpc( url=url, + flush_timeout_sec=flush_timeout_sec, default_blueprint=blueprint_storage, recording=recording.to_native() if recording is not None else None, ) From 3b18ec4b1a919d53356edc8d7ca93d330022cd26 Mon Sep 17 00:00:00 2001 From: jprochazk Date: Mon, 10 Feb 2025 22:57:49 +0100 Subject: [PATCH 86/87] fmt --- rerun_cpp/src/rerun/c/rerun.h | 3 ++- rerun_cpp/src/rerun/recording_stream.cpp | 7 ++++++- rerun_cpp/src/rerun/recording_stream.hpp | 4 +++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/rerun_cpp/src/rerun/c/rerun.h b/rerun_cpp/src/rerun/c/rerun.h index 1d097fe56a89..6ca090cb2de1 100644 --- a/rerun_cpp/src/rerun/c/rerun.h +++ b/rerun_cpp/src/rerun/c/rerun.h @@ -462,7 +462,8 @@ extern void rr_recording_stream_connect_grpc( /// dropping data if progress is not being made. Passing a negative value indicates no timeout, /// and can cause a call to `flush` to block indefinitely. extern void rr_recording_stream_spawn( - rr_recording_stream stream, const rr_spawn_options* spawn_opts, float flush_timeout_sec, rr_error* error + rr_recording_stream stream, const rr_spawn_options* spawn_opts, float flush_timeout_sec, + rr_error* error ); /// Stream all log-data to a given `.rrd` file. diff --git a/rerun_cpp/src/rerun/recording_stream.cpp b/rerun_cpp/src/rerun/recording_stream.cpp index 35f33a64d757..c67729f7aebc 100644 --- a/rerun_cpp/src/rerun/recording_stream.cpp +++ b/rerun_cpp/src/rerun/recording_stream.cpp @@ -129,7 +129,12 @@ namespace rerun { Error RecordingStream::connect_grpc(std::string_view url, float flush_timeout_sec) const { rr_error status = {}; - rr_recording_stream_connect_grpc(_id, detail::to_rr_string(url), flush_timeout_sec, &status); + rr_recording_stream_connect_grpc( + _id, + detail::to_rr_string(url), + flush_timeout_sec, + &status + ); return status; } diff --git a/rerun_cpp/src/rerun/recording_stream.hpp b/rerun_cpp/src/rerun/recording_stream.hpp index 22ee016c84d7..858d4cf41e84 100644 --- a/rerun_cpp/src/rerun/recording_stream.hpp +++ b/rerun_cpp/src/rerun/recording_stream.hpp @@ -171,7 +171,9 @@ namespace rerun { /// timeout, and can cause a call to `flush` to block indefinitely. /// /// This function returns immediately. - Error connect_grpc(std::string_view url = "http://127.0.0.1:9876", float flush_timeout_sec = 2.0) const; + Error connect_grpc( + std::string_view url = "http://127.0.0.1:9876", float flush_timeout_sec = 2.0 + ) const; /// Spawns a new Rerun Viewer process from an executable available in PATH, then connects to it /// over gRPC. From 3a4bdcc64e2d928fdd739491c570d67db9e2104a Mon Sep 17 00:00:00 2001 From: jprochazk Date: Mon, 10 Feb 2025 23:06:31 +0100 Subject: [PATCH 87/87] fix doc --- crates/top/re_sdk/src/log_sink.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/top/re_sdk/src/log_sink.rs b/crates/top/re_sdk/src/log_sink.rs index 47a78fcc05db..f5efd5dd3c55 100644 --- a/crates/top/re_sdk/src/log_sink.rs +++ b/crates/top/re_sdk/src/log_sink.rs @@ -342,7 +342,7 @@ pub struct GrpcSink { impl GrpcSink { /// Connect to the in-memory storage node over HTTP. /// - /// `flush_timeout` is the minimum time the [`TcpSink`] will wait during a flush + /// `flush_timeout` is the minimum time the [`GrpcSink`] will wait during a flush /// before potentially dropping data. Note: Passing `None` here can cause a /// call to `flush` to block indefinitely if a connection cannot be established. ///