Add support for previewing images in room scrollback (#108)

This commit is contained in:
Benjamin Grosse 2023-11-16 08:36:22 -08:00 committed by GitHub
parent 974775b29b
commit 221faa828d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 588 additions and 29 deletions

View file

@ -11,6 +11,7 @@ use std::sync::Arc;
use std::time::{Duration, Instant};
use emojis::Emoji;
use ratatui_image::picker::{Picker, ProtocolType};
use serde::{
de::Error as SerdeError,
de::Visitor,
@ -81,6 +82,9 @@ use modalkit::{
},
};
use crate::config::ImagePreviewProtocolValues;
use crate::message::ImageStatus;
use crate::preview::{source_from_event, spawn_insert_preview};
use crate::{
message::{Message, MessageEvent, MessageKey, MessageTimeStamp, Messages},
worker::Requester,
@ -617,6 +621,14 @@ pub enum IambError {
/// A failure to access the system's clipboard.
#[error("Could not use system clipboard data")]
Clipboard,
/// An failure during disk/network/ipc/etc. I/O.
#[error("Input/Output error: {0}")]
IOError(#[from] std::io::Error),
/// A failure while trying to show an image preview.
#[error("Preview error: {0}")]
Preview(String),
}
impl From<IambError> for UIError<IambInfo> {
@ -737,6 +749,11 @@ impl RoomInfo {
self.messages.get(self.get_message_key(event_id)?)
}
/// Get an event for an identifier as mutable.
pub fn get_event_mut(&mut self, event_id: &EventId) -> Option<&mut Message> {
self.messages.get_mut(self.keys.get(event_id)?.to_message_key()?)
}
/// Insert a reaction to a message.
pub fn insert_reaction(&mut self, react: ReactionEvent) {
match react {
@ -827,6 +844,37 @@ impl RoomInfo {
}
}
/// Insert a new message event, and spawn a task for image-preview if it has an image
/// attachment.
pub fn insert_with_preview(
&mut self,
room_id: OwnedRoomId,
store: AsyncProgramStore,
picker: Option<Picker>,
ev: RoomMessageEvent,
settings: &mut ApplicationSettings,
media: matrix_sdk::Media,
) {
let source = picker.and_then(|_| source_from_event(&ev));
self.insert(ev);
if let Some((event_id, source)) = source {
if let (Some(msg), Some(image_preview)) =
(self.get_event_mut(&event_id), &settings.tunables.image_preview)
{
msg.image_preview = ImageStatus::Downloading(image_preview.size.clone());
spawn_insert_preview(
store,
room_id,
event_id,
source,
media,
settings.dirs.image_previews.clone(),
)
}
}
}
/// Indicates whether we've recently fetched scrollback for this room.
pub fn recently_fetched(&self) -> bool {
self.fetch_last.map_or(false, |i| i.elapsed() < ROOM_FETCH_DEBOUNCE)
@ -936,6 +984,51 @@ fn emoji_map() -> CompletionMap<String, &'static Emoji> {
return emojis;
}
#[cfg(unix)]
fn picker_from_termios(protocol_type: Option<ProtocolType>) -> Option<Picker> {
let mut picker = match Picker::from_termios() {
Ok(picker) => picker,
Err(e) => {
tracing::error!("Failed to setup image previews: {e}");
return None;
},
};
if let Some(protocol_type) = protocol_type {
picker.protocol_type = protocol_type;
} else {
picker.guess_protocol();
}
Some(picker)
}
/// Windows cannot guess the right protocol, and always needs type and font_size.
#[cfg(windows)]
fn picker_from_termios(_: Option<ProtocolType>) -> Option<Picker> {
tracing::error!("\"image_preview\" requires \"protocol\" with \"type\" and \"font_size\" options on Windows.");
None
}
fn picker_from_settings(settings: &ApplicationSettings) -> Option<Picker> {
let image_preview = settings.tunables.image_preview.as_ref()?;
let image_preview_protocol = image_preview.protocol.as_ref();
if let Some(&ImagePreviewProtocolValues {
r#type: Some(protocol_type),
font_size: Some(font_size),
}) = image_preview_protocol
{
// User forced type and font_size: use that.
let mut picker = Picker::new(font_size);
picker.protocol_type = protocol_type;
Some(picker)
} else {
// Guess, but use type if forced.
picker_from_termios(image_preview_protocol.and_then(|p| p.r#type))
}
}
/// Information gathered during server syncs about joined rooms.
#[derive(Default)]
pub struct SyncInfo {
@ -980,15 +1073,20 @@ pub struct ChatStore {
/// Information gathered by the background thread.
pub sync_info: SyncInfo,
/// Image preview "protocol" picker.
pub picker: Option<Picker>,
}
impl ChatStore {
/// Create a new [ChatStore].
pub fn new(worker: Requester, settings: ApplicationSettings) -> Self {
let picker = picker_from_settings(&settings);
ChatStore {
worker,
settings,
picker,
cmds: crate::commands::setup_commands(),
emojis: emoji_map(),

View file

@ -11,6 +11,7 @@ use std::process;
use clap::Parser;
use matrix_sdk::ruma::{OwnedRoomAliasId, OwnedRoomId, OwnedUserId, UserId};
use ratatui_image::picker::ProtocolType;
use serde::{de::Error as SerdeError, de::Visitor, Deserialize, Deserializer};
use tracing::Level;
use url::Url;
@ -266,6 +267,45 @@ pub enum UserDisplayStyle {
DisplayName,
}
#[derive(Clone)]
pub struct ImagePreviewValues {
pub size: ImagePreviewSize,
pub protocol: Option<ImagePreviewProtocolValues>,
}
#[derive(Clone, Default, Deserialize)]
pub struct ImagePreview {
pub size: Option<ImagePreviewSize>,
pub protocol: Option<ImagePreviewProtocolValues>,
}
impl ImagePreview {
fn values(self) -> ImagePreviewValues {
ImagePreviewValues {
size: self.size.unwrap_or_default(),
protocol: self.protocol,
}
}
}
#[derive(Clone, Deserialize)]
pub struct ImagePreviewSize {
pub width: usize,
pub height: usize,
}
impl Default for ImagePreviewSize {
fn default() -> Self {
ImagePreviewSize { width: 66, height: 10 }
}
}
#[derive(Clone, Deserialize)]
pub struct ImagePreviewProtocolValues {
pub r#type: Option<ProtocolType>,
pub font_size: Option<(u16, u16)>,
}
#[derive(Clone)]
pub struct SortValues {
pub dms: Vec<SortColumn<SortFieldRoom>>,
@ -308,6 +348,7 @@ pub struct TunableValues {
pub username_display: UserDisplayStyle,
pub default_room: Option<String>,
pub open_command: Option<Vec<String>>,
pub image_preview: Option<ImagePreviewValues>,
}
#[derive(Clone, Default, Deserialize)]
@ -326,6 +367,7 @@ pub struct Tunables {
pub username_display: Option<UserDisplayStyle>,
pub default_room: Option<String>,
pub open_command: Option<Vec<String>>,
pub image_preview: Option<ImagePreview>,
}
impl Tunables {
@ -346,6 +388,7 @@ impl Tunables {
username_display: self.username_display.or(other.username_display),
default_room: self.default_room.or(other.default_room),
open_command: self.open_command.or(other.open_command),
image_preview: self.image_preview.or(other.image_preview),
}
}
@ -364,6 +407,7 @@ impl Tunables {
username_display: self.username_display.unwrap_or_default(),
default_room: self.default_room,
open_command: self.open_command,
image_preview: self.image_preview.map(ImagePreview::values),
}
}
}
@ -373,6 +417,7 @@ pub struct DirectoryValues {
pub cache: PathBuf,
pub logs: PathBuf,
pub downloads: Option<PathBuf>,
pub image_previews: PathBuf,
}
#[derive(Clone, Default, Deserialize)]
@ -380,6 +425,7 @@ pub struct Directories {
pub cache: Option<PathBuf>,
pub logs: Option<PathBuf>,
pub downloads: Option<PathBuf>,
pub image_previews: Option<PathBuf>,
}
impl Directories {
@ -388,6 +434,7 @@ impl Directories {
cache: self.cache.or(other.cache),
logs: self.logs.or(other.logs),
downloads: self.downloads.or(other.downloads),
image_previews: self.image_previews.or(other.image_previews),
}
}
@ -409,7 +456,13 @@ impl Directories {
let downloads = self.downloads.or_else(dirs::download_dir);
DirectoryValues { cache, logs, downloads }
let image_previews = self.image_previews.unwrap_or_else(|| {
let mut dir = cache.clone();
dir.push("image_preview_downloads");
dir
});
DirectoryValues { cache, logs, downloads, image_previews }
}
}

View file

@ -68,6 +68,7 @@ mod commands;
mod config;
mod keybindings;
mod message;
mod preview;
mod util;
mod windows;
mod worker;
@ -267,6 +268,7 @@ impl Application {
let screen = setup_screen(settings, locked.deref_mut())?;
let worker = locked.application.worker.clone();
drop(locked);
let actstack = VecDeque::new();
@ -786,6 +788,7 @@ fn main() -> IambResult<()> {
let log_dir = settings.dirs.logs.as_path();
create_dir_all(settings.matrix_dir.as_path())?;
create_dir_all(settings.dirs.image_previews.as_path())?;
create_dir_all(log_dir)?;
let appender = tracing_appender::rolling::daily(log_dir, log_prefix);

View file

@ -51,7 +51,9 @@ use modalkit::tui::{
};
use modalkit::editing::{base::ViewportContext, cursor::Cursor};
use ratatui_image::protocol::Protocol;
use crate::config::ImagePreviewSize;
use crate::{
base::{IambResult, RoomInfo},
config::ApplicationSettings,
@ -585,12 +587,20 @@ impl<'a> MessageFormatter<'a> {
}
}
pub enum ImageStatus {
None,
Downloading(ImagePreviewSize),
Loaded(Box<dyn Protocol>),
Error(String),
}
pub struct Message {
pub event: MessageEvent,
pub sender: OwnedUserId,
pub timestamp: MessageTimeStamp,
pub downloaded: bool,
pub html: Option<StyleTree>,
pub image_preview: ImageStatus,
}
impl Message {
@ -598,7 +608,14 @@ impl Message {
let html = event.html();
let downloaded = false;
Message { event, sender, timestamp, downloaded, html }
Message {
event,
sender,
timestamp,
downloaded,
html,
image_preview: ImageStatus::None,
}
}
pub fn reply_to(&self) -> Option<OwnedEventId> {
@ -681,6 +698,36 @@ impl Message {
}
}
/// Get the image preview Protocol and x,y offset, based on get_render_format.
pub fn line_preview<'a>(
&'a self,
prev: Option<&Message>,
vwctx: &ViewportContext<MessageCursor>,
info: &'a RoomInfo,
) -> Option<(&dyn Protocol, u16, u16)> {
let width = vwctx.get_width();
// The x position where get_render_format would render the text.
let x = (if USER_GUTTER + MIN_MSG_LEN <= width {
USER_GUTTER
} else {
0
} + 1) as u16;
// See get_render_format; account for possible "date" line.
let date_y = match &prev {
Some(prev) if !prev.timestamp.same_day(&self.timestamp) => 1,
_ => 0,
};
if let ImageStatus::Loaded(backend) = &self.image_preview {
return Some((backend.as_ref(), x, date_y));
} else if let Some(reply) = self.reply_to().and_then(|e| info.get_event(&e)) {
if let ImageStatus::Loaded(backend) = &reply.image_preview {
// The reply should be offset a bit:
return Some((backend.as_ref(), x + 2, date_y + 1));
}
}
None
}
pub fn show<'a>(
&'a self,
prev: Option<&Message>,
@ -791,6 +838,19 @@ impl Message {
msg.to_mut().push_str(" \u{2705}");
}
if let Some(placeholder) = match &self.image_preview {
ImageStatus::None => None,
ImageStatus::Downloading(image_preview_size) => {
Some(Message::placeholder_frame(Some("Downloading..."), image_preview_size))
},
ImageStatus::Loaded(backend) => {
Some(Message::placeholder_frame(None, &backend.rect().into()))
},
ImageStatus::Error(err) => Some(format!("[Image error: {err}]\n")),
} {
msg.to_mut().insert_str(0, &placeholder);
}
wrapped_text(msg, width, style)
}
}
@ -803,6 +863,24 @@ impl Message {
settings.get_user_span(self.sender.as_ref(), info)
}
/// Before the image is loaded, already display a placeholder frame of the image size.
fn placeholder_frame(text: Option<&str>, image_preview_size: &ImagePreviewSize) -> String {
let ImagePreviewSize { width, height } = image_preview_size;
let mut placeholder = "\u{230c}".to_string();
placeholder.push_str(&" ".repeat(width - 2));
placeholder.push_str("\u{230d}\n");
placeholder.push(' ');
if let Some(text) = text {
placeholder.push_str(text);
}
placeholder.push_str(&"\n".repeat(height - 2));
placeholder.push('\u{230e}');
placeholder.push_str(&" ".repeat(width - 2));
placeholder.push_str("\u{230f}\n");
placeholder
}
fn show_sender<'a>(
&'a self,
prev: Option<&Message>,

172
src/preview.rs Normal file
View file

@ -0,0 +1,172 @@
use std::{
fs::File,
io::{Read, Write},
path::{Path, PathBuf},
};
use matrix_sdk::{
media::{MediaFormat, MediaRequest},
ruma::{
events::{
room::{
message::{MessageType, RoomMessageEventContent},
MediaSource,
},
MessageLikeEvent,
},
OwnedEventId,
OwnedRoomId,
},
Media,
};
use modalkit::tui::layout::Rect;
use ratatui_image::Resize;
use crate::{
base::{AsyncProgramStore, ChatStore, IambError},
config::ImagePreviewSize,
message::ImageStatus,
};
pub fn source_from_event(
ev: &MessageLikeEvent<RoomMessageEventContent>,
) -> Option<(OwnedEventId, MediaSource)> {
if let MessageLikeEvent::Original(ev) = &ev {
if let MessageType::Image(c) = &ev.content.msgtype {
return Some((ev.event_id.clone(), c.source.clone()));
}
}
None
}
impl From<ImagePreviewSize> for Rect {
fn from(value: ImagePreviewSize) -> Self {
Rect::new(0, 0, value.width as _, value.height as _)
}
}
impl From<Rect> for ImagePreviewSize {
fn from(rect: Rect) -> Self {
ImagePreviewSize { width: rect.width as _, height: rect.height as _ }
}
}
/// Download and prepare the preview, and then lock the store to insert it.
pub fn spawn_insert_preview(
store: AsyncProgramStore,
room_id: OwnedRoomId,
event_id: OwnedEventId,
source: MediaSource,
media: Media,
cache_dir: PathBuf,
) {
tokio::spawn(async move {
let img = download_or_load(event_id.to_owned(), source, media, cache_dir)
.await
.map(std::io::Cursor::new)
.map(image::io::Reader::new)
.map_err(IambError::Matrix)
.and_then(|reader| reader.with_guessed_format().map_err(IambError::IOError))
.and_then(|reader| reader.decode().map_err(IambError::Image));
match img {
Err(err) => {
try_set_msg_preview_error(
&mut store.lock().await.application,
room_id,
event_id,
err,
);
},
Ok(img) => {
let mut locked = store.lock().await;
let ChatStore { rooms, picker, settings, .. } = &mut locked.application;
match picker
.as_mut()
.ok_or_else(|| IambError::Preview("Picker is empty".to_string()))
.and_then(|picker| {
Ok((
picker,
rooms
.get_or_default(room_id.clone())
.get_event_mut(&event_id)
.ok_or_else(|| {
IambError::Preview("Message not found".to_string())
})?,
settings.tunables.image_preview.clone().ok_or_else(|| {
IambError::Preview("image_preview settings not found".to_string())
})?,
))
})
.and_then(|(picker, msg, image_preview)| {
picker
.new_protocol(img, image_preview.size.into(), Resize::Fit)
.map_err(|err| IambError::Preview(format!("{err:?}")))
.map(|backend| (backend, msg))
}) {
Err(err) => {
try_set_msg_preview_error(&mut locked.application, room_id, event_id, err);
},
Ok((backend, msg)) => {
msg.image_preview = ImageStatus::Loaded(backend);
},
}
},
}
});
}
fn try_set_msg_preview_error(
application: &mut ChatStore,
room_id: OwnedRoomId,
event_id: OwnedEventId,
err: IambError,
) {
let rooms = &mut application.rooms;
match rooms
.get_or_default(room_id.clone())
.get_event_mut(&event_id)
.ok_or_else(|| IambError::Preview("Message not found".to_string()))
{
Ok(msg) => msg.image_preview = ImageStatus::Error(format!("{err:?}")),
Err(err) => {
tracing::error!(
"Failed to set error on msg.image_backend for event {}, room {}: {}",
event_id,
room_id,
err
)
},
}
}
async fn download_or_load(
event_id: OwnedEventId,
source: MediaSource,
media: Media,
mut cache_path: PathBuf,
) -> Result<Vec<u8>, matrix_sdk::Error> {
cache_path.push(Path::new(event_id.localpart()));
match File::open(&cache_path) {
Ok(mut f) => {
let mut buffer = Vec::new();
f.read_to_end(&mut buffer)?;
Ok(buffer)
},
Err(_) => {
media
.get_media_content(&MediaRequest { source, format: MediaFormat::File }, true)
.await
.and_then(|buffer| {
if let Err(err) =
File::create(&cache_path).and_then(|mut f| f.write_all(&buffer))
{
return Err(err.into());
}
Ok(buffer)
})
},
}
}

View file

@ -172,6 +172,7 @@ pub fn mock_dirs() -> DirectoryValues {
cache: PathBuf::new(),
logs: PathBuf::new(),
downloads: None,
image_previews: PathBuf::new(),
}
}
@ -195,6 +196,7 @@ pub fn mock_tunables() -> TunableValues {
.collect::<HashMap<_, _>>(),
open_command: None,
username_display: UserDisplayStyle::Username,
image_preview: None,
}
}

View file

@ -1,6 +1,7 @@
//! Message scrollback
use std::collections::HashSet;
use ratatui_image::FixedImage;
use regex::Regex;
use matrix_sdk::ruma::OwnedRoomId;
@ -1264,6 +1265,7 @@ impl<'a> StatefulWidget for Scrollback<'a> {
fn render(self, area: Rect, buf: &mut Buffer, state: &mut Self::State) {
let info = self.store.application.rooms.get_or_default(state.room_id.clone());
let settings = &self.store.application.settings;
let picker = &self.store.application.picker;
let area = if state.cursor.timestamp.is_some() {
render_jump_to_recent(area, buf, self.focused)
} else {
@ -1307,7 +1309,11 @@ impl<'a> StatefulWidget for Scrollback<'a> {
let sel = key == cursor_key;
let txt = item.show(prev, foc && sel, &state.viewctx, info, settings);
prev = Some(item);
let mut msg_preview = if picker.is_some() {
item.line_preview(prev, &state.viewctx, info)
} else {
None
};
let incomplete_ok = !full || !sel;
@ -1323,9 +1329,17 @@ impl<'a> StatefulWidget for Scrollback<'a> {
continue;
}
lines.push((key, row, line));
let line_preview = match msg_preview {
// Only take the preview into the matching row number.
Some((_, _, y)) if y as usize == row => msg_preview.take(),
_ => None,
};
lines.push((key, row, line, line_preview));
sawit |= sel;
}
prev = Some(item);
}
if lines.len() > height {
@ -1333,7 +1347,7 @@ impl<'a> StatefulWidget for Scrollback<'a> {
let _ = lines.drain(..n);
}
if let Some(((ts, event_id), row, _)) = lines.first() {
if let Some(((ts, event_id), row, _, _)) = lines.first() {
state.viewctx.corner.timestamp = Some((*ts, event_id.clone()));
state.viewctx.corner.text_row = *row;
}
@ -1341,11 +1355,27 @@ impl<'a> StatefulWidget for Scrollback<'a> {
let mut y = area.top();
let x = area.left();
for (_, _, txt) in lines.into_iter() {
let mut image_previews = vec![];
for ((_, _), _, txt, line_preview) in lines.into_iter() {
let _ = buf.set_line(x, y, &txt, area.width);
if let Some((backend, msg_x, _)) = line_preview {
image_previews.push((x + msg_x, y, backend));
}
y += 1;
}
// Render image previews after all text lines have been drawn, as the render might draw below the current
// line.
for (x, y, backend) in image_previews {
let image_widget = FixedImage::new(backend);
let mut rect = backend.rect();
rect.x = x;
rect.y = y;
// Don't render outside of scrollback area
if rect.bottom() <= area.bottom() && rect.right() <= area.right() {
image_widget.render(rect, buf);
}
}
if self.room_focused &&
settings.tunables.read_receipt_send &&

View file

@ -244,16 +244,20 @@ async fn load_older_one(
}
}
async fn load_insert(
client: &Client,
room_id: OwnedRoomId,
res: MessageFetchResult,
store: AsyncProgramStore,
) {
async fn load_insert(room_id: OwnedRoomId, res: MessageFetchResult, store: AsyncProgramStore) {
let mut locked = store.lock().await;
let ChatStore { need_load, presences, rooms, .. } = &mut locked.application;
let ChatStore {
need_load,
presences,
rooms,
worker,
picker,
settings,
..
} = &mut locked.application;
let info = rooms.get_or_default(room_id.clone());
info.fetching = false;
let client = &worker.client;
match res {
Ok((fetch_id, msgs)) => {
@ -270,7 +274,14 @@ async fn load_insert(
info.insert_encrypted(msg);
},
AnyMessageLikeEvent::RoomMessage(msg) => {
info.insert(msg);
info.insert_with_preview(
room_id.clone(),
store.clone(),
*picker,
msg,
settings,
client.media(),
);
},
AnyMessageLikeEvent::Reaction(ev) => {
info.insert_reaction(ev);
@ -298,12 +309,11 @@ async fn load_older(client: &Client, store: &AsyncProgramStore) -> usize {
.await
.into_iter()
.map(|(room_id, fetch_id)| {
let client = client.clone();
let store = store.clone();
async move {
let res = load_older_one(&client, room_id.as_ref(), fetch_id, limit).await;
load_insert(&client, room_id, res, store).await;
let res = load_older_one(client, room_id.as_ref(), fetch_id, limit).await;
load_insert(room_id, res, store).await;
}
})
.collect::<FuturesUnordered<_>>()
@ -813,9 +823,20 @@ impl ClientWorker {
let sender = ev.sender().to_owned();
let _ = locked.application.presences.get_or_default(sender);
let info = locked.application.get_room_info(room_id.to_owned());
let ChatStore { rooms, picker, settings, .. } = &mut locked.application;
let info = rooms.get_or_default(room_id.to_owned());
update_event_receipts(info, &room, ev.event_id()).await;
info.insert(ev.into_full_event(room_id.to_owned()));
let full_ev = ev.into_full_event(room_id.to_owned());
info.insert_with_preview(
room_id.to_owned(),
store.clone(),
*picker,
full_ev,
settings,
client.media(),
);
}
},
);