From 17fd4dd5cad2c8e294faf79f24c16b6ed69a581b Mon Sep 17 00:00:00 2001 From: Stefan Ceriu Date: Tue, 2 May 2023 15:54:01 +0300 Subject: [PATCH] ffi: Support sending image attachments through the timeline --- bindings/matrix-sdk-ffi/src/error.rs | 24 ++++ bindings/matrix-sdk-ffi/src/room.rs | 134 ++++++++++++++++++++- bindings/matrix-sdk-ffi/src/timeline.rs | 94 ++++++++++++++- crates/matrix-sdk/src/room/timeline/mod.rs | 53 +++++++- 4 files changed, 300 insertions(+), 5 deletions(-) diff --git a/bindings/matrix-sdk-ffi/src/error.rs b/bindings/matrix-sdk-ffi/src/error.rs index ee9f01bdce0..ab9bf678d6a 100644 --- a/bindings/matrix-sdk-ffi/src/error.rs +++ b/bindings/matrix-sdk-ffi/src/error.rs @@ -59,3 +59,27 @@ impl From for ClientError { anyhow::Error::from(e).into() } } + +#[derive(Debug, thiserror::Error, uniffi::Error)] +#[uniffi(flat_error)] +pub enum RoomError { + #[error("Invalid attachment data")] + InvalidAttachmentData, + #[error("Invalid attachment mime type")] + InvalidAttachmentMimeType, + #[error("Timeline unavailable")] + TimelineUnavailable, + #[error("Invalid thumbnail data")] + InvalidThumbnailData, + #[error("Failed sending attachment")] + FailedSendingAttachment, +} + +#[derive(Debug, thiserror::Error, uniffi::Error)] +#[uniffi(flat_error)] +pub enum TimelineError { + #[error("Required value missing from the media info")] + MissingMediaInfoField, + #[error("Media info field invalid")] + InvalidMediaInfoField, +} diff --git a/bindings/matrix-sdk-ffi/src/room.rs b/bindings/matrix-sdk-ffi/src/room.rs index 1b412cc110b..3faab9e4497 100644 --- a/bindings/matrix-sdk-ffi/src/room.rs +++ b/bindings/matrix-sdk-ffi/src/room.rs @@ -1,11 +1,16 @@ use std::{ convert::TryFrom, + fs, sync::{Arc, RwLock}, }; use anyhow::{anyhow, bail, Context, Result}; use futures_util::StreamExt; use matrix_sdk::{ + attachment::{ + AttachmentConfig, AttachmentInfo, BaseAudioInfo, BaseFileInfo, BaseImageInfo, + BaseThumbnailInfo, BaseVideoInfo, Thumbnail, + }, room::{timeline::Timeline, Receipts, Room as SdkRoom}, ruma::{ api::client::{receipt::create_receipt::v3::ReceiptType, room::report_content}, @@ -25,7 +30,11 @@ use mime::Mime; use tracing::error; use super::RUNTIME; -use crate::{error::ClientError, RoomMember, TimelineDiff, TimelineItem, TimelineListener}; +use crate::{ + error::{ClientError, RoomError}, + AudioInfo, FileInfo, ImageInfo, RoomMember, ThumbnailInfo, TimelineDiff, TimelineItem, + TimelineListener, VideoInfo, +}; #[derive(uniffi::Enum)] pub enum Membership { @@ -637,6 +646,129 @@ impl Room { Ok(()) }) } + + pub fn send_image( + &self, + url: String, + thumbnail_url: String, + image_info: ImageInfo, + ) -> Result<(), RoomError> { + let mime_str = image_info.mimetype.as_ref().ok_or(RoomError::InvalidAttachmentMimeType)?; + let mime_type = + mime_str.parse::().map_err(|_| RoomError::InvalidAttachmentMimeType)?; + + let base_image_info = + BaseImageInfo::try_from(&image_info).map_err(|_| RoomError::InvalidAttachmentData)?; + + let attachment_info = AttachmentInfo::Image(base_image_info); + + let attachment_config = match image_info.thumbnail_info { + Some(thumbnail_image_info) => { + let thumbnail = self.build_thumbnail_info(thumbnail_url, thumbnail_image_info)?; + AttachmentConfig::with_thumbnail(thumbnail).info(attachment_info) + } + None => AttachmentConfig::new().info(attachment_info), + }; + + self.send_attachment(url, mime_type, attachment_config) + } + + pub fn send_video( + &self, + url: String, + thumbnail_url: String, + video_info: VideoInfo, + ) -> Result<(), RoomError> { + let mime_str = video_info.mimetype.as_ref().ok_or(RoomError::InvalidAttachmentMimeType)?; + let mime_type = + mime_str.parse::().map_err(|_| RoomError::InvalidAttachmentMimeType)?; + + let base_video_info: BaseVideoInfo = + BaseVideoInfo::try_from(&video_info).map_err(|_| RoomError::InvalidAttachmentData)?; + + let attachment_info = AttachmentInfo::Video(base_video_info); + + let attachment_config = match video_info.thumbnail_info { + Some(thumbnail_image_info) => { + let thumbnail = self.build_thumbnail_info(thumbnail_url, thumbnail_image_info)?; + AttachmentConfig::with_thumbnail(thumbnail).info(attachment_info) + } + None => AttachmentConfig::new().info(attachment_info), + }; + + self.send_attachment(url, mime_type, attachment_config) + } + + pub fn send_audio(&self, url: String, audio_info: AudioInfo) -> Result<(), RoomError> { + let mime_str = audio_info.mimetype.as_ref().ok_or(RoomError::InvalidAttachmentMimeType)?; + let mime_type = + mime_str.parse::().map_err(|_| RoomError::InvalidAttachmentMimeType)?; + + let base_audio_info: BaseAudioInfo = + BaseAudioInfo::try_from(&audio_info).map_err(|_| RoomError::InvalidAttachmentData)?; + + let attachment_info = AttachmentInfo::Audio(base_audio_info); + let attachment_config = AttachmentConfig::new().info(attachment_info); + + self.send_attachment(url, mime_type, attachment_config) + } + + pub fn send_file(&self, url: String, file_info: FileInfo) -> Result<(), RoomError> { + let mime_str = file_info.mimetype.as_ref().ok_or(RoomError::InvalidAttachmentMimeType)?; + let mime_type = + mime_str.parse::().map_err(|_| RoomError::InvalidAttachmentMimeType)?; + + let base_file_info: BaseFileInfo = + BaseFileInfo::try_from(&file_info).map_err(|_| RoomError::InvalidAttachmentData)?; + + let attachment_info = AttachmentInfo::File(base_file_info); + let attachment_config = AttachmentConfig::new().info(attachment_info); + + self.send_attachment(url, mime_type, attachment_config) + } +} + +impl Room { + fn build_thumbnail_info( + &self, + thumbnail_url: String, + thumbnail_info: ThumbnailInfo, + ) -> Result { + let thumbnail_data = + fs::read(thumbnail_url).map_err(|_| RoomError::InvalidThumbnailData)?; + + let base_thumbnail_info = BaseThumbnailInfo::try_from(&thumbnail_info) + .map_err(|_| RoomError::InvalidAttachmentData)?; + + let mime_str = + thumbnail_info.mimetype.as_ref().ok_or(RoomError::InvalidAttachmentMimeType)?; + let mime_type = + mime_str.parse::().map_err(|_| RoomError::InvalidAttachmentMimeType)?; + + Ok(Thumbnail { + data: thumbnail_data, + content_type: mime_type, + info: Some(base_thumbnail_info), + }) + } + + fn send_attachment( + &self, + url: String, + mime_type: Mime, + attachment_config: AttachmentConfig, + ) -> Result<(), RoomError> { + let timeline_guard = self.timeline.read().unwrap(); + let timeline = timeline_guard.as_ref().ok_or(RoomError::TimelineUnavailable)?; + + RUNTIME.block_on(async move { + timeline + .send_attachment(url, mime_type, attachment_config) + .await + .map_err(|_| RoomError::FailedSendingAttachment)?; + Ok(()) + }) + } } impl std::ops::Deref for Room { diff --git a/bindings/matrix-sdk-ffi/src/timeline.rs b/bindings/matrix-sdk-ffi/src/timeline.rs index 67d5ce8f0c8..d42bc52aa0c 100644 --- a/bindings/matrix-sdk-ffi/src/timeline.rs +++ b/bindings/matrix-sdk-ffi/src/timeline.rs @@ -1,13 +1,17 @@ -use std::sync::Arc; +use std::{sync::Arc, time::Duration}; use anyhow::bail; use extension_trait::extension_trait; use eyeball_im::VectorDiff; -use matrix_sdk::room::timeline::{Profile, TimelineDetails}; pub use matrix_sdk::ruma::events::room::{message::RoomMessageEventContent, MediaSource}; +use matrix_sdk::{ + attachment::{BaseAudioInfo, BaseFileInfo, BaseImageInfo, BaseThumbnailInfo, BaseVideoInfo}, + room::timeline::{Profile, TimelineDetails}, +}; +use ruma::UInt; use tracing::warn; -use crate::helpers::unwrap_or_clone_arc; +use crate::{error::TimelineError, helpers::unwrap_or_clone_arc}; #[uniffi::export] pub fn media_source_from_url(url: String) -> Arc { @@ -582,6 +586,27 @@ pub struct ImageInfo { pub blurhash: Option, } +impl TryFrom<&ImageInfo> for BaseImageInfo { + type Error = TimelineError; + + fn try_from(value: &ImageInfo) -> Result { + let height = UInt::try_from(value.height.ok_or(TimelineError::MissingMediaInfoField)?) + .map_err(|_| TimelineError::InvalidMediaInfoField)?; + let width = UInt::try_from(value.width.ok_or(TimelineError::MissingMediaInfoField)?) + .map_err(|_| TimelineError::InvalidMediaInfoField)?; + let size = UInt::try_from(value.size.ok_or(TimelineError::MissingMediaInfoField)?) + .map_err(|_| TimelineError::InvalidMediaInfoField)?; + let blurhash = value.blurhash.clone().ok_or(TimelineError::MissingMediaInfoField)?; + + Ok(BaseImageInfo { + height: Some(height), + width: Some(width), + size: Some(size), + blurhash: Some(blurhash), + }) + } +} + #[derive(Clone, uniffi::Record)] pub struct AudioInfo { // FIXME: duration should be a std::time::Duration once the UniFFI proc-macro API adds support @@ -591,6 +616,19 @@ pub struct AudioInfo { pub mimetype: Option, } +impl TryFrom<&AudioInfo> for BaseAudioInfo { + type Error = TimelineError; + + fn try_from(value: &AudioInfo) -> Result { + let duration = + value.duration.map(Duration::from_secs).ok_or(TimelineError::MissingMediaInfoField)?; + let size = UInt::try_from(value.size.ok_or(TimelineError::MissingMediaInfoField)?) + .map_err(|_| TimelineError::InvalidMediaInfoField)?; + + Ok(BaseAudioInfo { duration: Some(duration), size: Some(size) }) + } +} + #[derive(Clone, uniffi::Record)] pub struct VideoInfo { pub duration: Option, @@ -603,6 +641,30 @@ pub struct VideoInfo { pub blurhash: Option, } +impl TryFrom<&VideoInfo> for BaseVideoInfo { + type Error = TimelineError; + + fn try_from(value: &VideoInfo) -> Result { + let duration = + value.duration.map(Duration::from_secs).ok_or(TimelineError::MissingMediaInfoField)?; + let height = UInt::try_from(value.height.ok_or(TimelineError::MissingMediaInfoField)?) + .map_err(|_| TimelineError::InvalidMediaInfoField)?; + let width = UInt::try_from(value.width.ok_or(TimelineError::MissingMediaInfoField)?) + .map_err(|_| TimelineError::InvalidMediaInfoField)?; + let size = UInt::try_from(value.size.ok_or(TimelineError::MissingMediaInfoField)?) + .map_err(|_| TimelineError::InvalidMediaInfoField)?; + let blurhash = value.blurhash.clone().ok_or(TimelineError::MissingMediaInfoField)?; + + Ok(BaseVideoInfo { + duration: Some(duration), + height: Some(height), + width: Some(width), + size: Some(size), + blurhash: Some(blurhash), + }) + } +} + #[derive(Clone, uniffi::Record)] pub struct FileInfo { pub mimetype: Option, @@ -611,6 +673,17 @@ pub struct FileInfo { pub thumbnail_source: Option>, } +impl TryFrom<&FileInfo> for BaseFileInfo { + type Error = TimelineError; + + fn try_from(value: &FileInfo) -> Result { + let size = UInt::try_from(value.size.ok_or(TimelineError::MissingMediaInfoField)?) + .map_err(|_| TimelineError::InvalidMediaInfoField)?; + + Ok(BaseFileInfo { size: Some(size) }) + } +} + #[derive(Clone, uniffi::Record)] pub struct ThumbnailInfo { pub height: Option, @@ -619,6 +692,21 @@ pub struct ThumbnailInfo { pub size: Option, } +impl TryFrom<&ThumbnailInfo> for BaseThumbnailInfo { + type Error = TimelineError; + + fn try_from(value: &ThumbnailInfo) -> Result { + let height = UInt::try_from(value.height.ok_or(TimelineError::MissingMediaInfoField)?) + .map_err(|_| TimelineError::InvalidMediaInfoField)?; + let width = UInt::try_from(value.width.ok_or(TimelineError::MissingMediaInfoField)?) + .map_err(|_| TimelineError::InvalidMediaInfoField)?; + let size = UInt::try_from(value.size.ok_or(TimelineError::MissingMediaInfoField)?) + .map_err(|_| TimelineError::InvalidMediaInfoField)?; + + Ok(BaseThumbnailInfo { height: Some(height), width: Some(width), size: Some(size) }) + } +} + #[derive(Clone, uniffi::Record)] pub struct NoticeMessageContent { pub body: String, diff --git a/crates/matrix-sdk/src/room/timeline/mod.rs b/crates/matrix-sdk/src/room/timeline/mod.rs index 36763777609..23b0b8ace76 100644 --- a/crates/matrix-sdk/src/room/timeline/mod.rs +++ b/crates/matrix-sdk/src/room/timeline/mod.rs @@ -16,11 +16,13 @@ //! //! See [`Timeline`] for details. -use std::{pin::Pin, sync::Arc, task::Poll}; +use std::{fs, path::Path, pin::Pin, sync::Arc, task::Poll}; use eyeball_im::{VectorDiff, VectorSubscriber}; use futures_core::Stream; +use futures_util::TryFutureExt; use imbl::Vector; +use mime::Mime; use pin_project_lite::pin_project; use ruma::{ api::client::receipt::create_receipt::v3::ReceiptType, @@ -37,6 +39,7 @@ use tracing::{error, instrument, warn}; use super::{Joined, Receipts}; use crate::{ + attachment::AttachmentConfig, event_handler::EventHandlerHandle, room::{self, MessagesOptions}, Client, Result, @@ -287,6 +290,42 @@ impl Timeline { self.inner.update_event_send_state(&txn_id, send_state).await; } + /// Sends an attachment to the room. It does not currently support local + /// echoes + /// + /// If the encryption feature is enabled, this method will transparently + /// encrypt the room message if the room is encrypted. + /// + /// # Arguments + /// + /// * `url` - The url for the file to be sent + /// + /// * `mime_type` - The attachment's mime type + /// + /// * `config` - An attachment configuration object containing details about + /// the attachment + /// like a thumbnail, its size, duration etc. + pub async fn send_attachment( + &self, + url: String, + mime_type: Mime, + config: AttachmentConfig, + ) -> Result<(), Error> { + // If this room isn't actually in joined state, we'll get a server error. + // Not ideal, but works for now. + let room = Joined { inner: self.room().clone() }; + + let body = + Path::new(&url).file_name().ok_or(Error::InvalidAttachmentFileName)?.to_str().unwrap(); + let data = fs::read(&url).map_err(|_| Error::InvalidAttachmentData)?; + + room.send_attachment(body, &mime_type, data, config) + .map_err(|_| Error::FailedSendingAttachment) + .await?; + + Ok(()) + } + /// Fetch unavailable details about the event with the given ID. /// /// This method only works for IDs of [`RemoteEventTimelineItem`]s, to @@ -577,6 +616,18 @@ pub enum Error { /// The event is currently unsupported for this use case. #[error("Unsupported event")] UnsupportedEvent, + + /// Couldn't read the attachment data from the given URL + #[error("Invalid attachment data")] + InvalidAttachmentData, + + /// The attachment file name used as a body is invalid + #[error("Invalid attachment file name")] + InvalidAttachmentFileName, + + /// The attachment could not be sent + #[error("Failed sending attachment")] + FailedSendingAttachment, } /// Result of comparing events position in the timeline.