Serialize images as base64 by rounding channels from floats to u8 (#1120)

Serialise images as base64

Cargo.lock

@@ -1587,6 +1587,7 @@ name = "graphene-core"
 version = "0.1.0"
 dependencies = [
  "async-trait",
+ "base64 0.13.1",
  "bezier-rs",
  "bytemuck",
  "dyn-any",

document-legacy/src/document.rs

@@ -2,7 +2,7 @@ use crate::boolean_ops::composite_boolean_operation;
 use crate::intersection::Quad;
 use crate::layers::folder_layer::FolderLayer;
 use crate::layers::layer_info::{Layer, LayerData, LayerDataType, LayerDataTypeDiscriminant};
-use crate::layers::nodegraph_layer::NodeGraphFrameLayer;
+use crate::layers::nodegraph_layer::{CachedOutputData, NodeGraphFrameLayer};
 use crate::layers::shape_layer::ShapeLayer;
 use crate::layers::style::RenderData;
 use crate::layers::text_layer::{Font, TextLayer};
@@ -570,16 +570,6 @@ impl Document {
 
 				Some([vec![DocumentChanged, CreatedLayer { path: path.clone() }], update_thumbnails_upstream(&path)].concat())
 			}
-			Operation::SetNodeGraphFrameImageData { layer_path, image_data } => {
-				let layer = self.layer_mut(&layer_path).expect("Setting NodeGraphFrame image data for invalid layer");
-				if let LayerDataType::NodeGraphFrame(node_graph_frame) = &mut layer.data {
-					let image_data = std::sync::Arc::new(image_data);
-					node_graph_frame.image_data = Some(crate::layers::nodegraph_layer::ImageData { image_data });
-				} else {
-					panic!("Incorrectly trying to set image data for a layer that is not an NodeGraphFrame layer type");
-				}
-				Some(vec![LayerChanged { path: layer_path.clone() }])
-			}
 			Operation::SetLayerPreserveAspect { layer_path, preserve_aspect } => {
 				if let Ok(layer) = self.layer_mut(&layer_path) {
 					layer.preserve_aspect = preserve_aspect;
@@ -781,15 +771,14 @@ impl Document {
 				self.mark_as_dirty(&path)?;
 				Some([vec![DocumentChanged], update_thumbnails_upstream(&path)].concat())
 			}
-			Operation::SetLayerBlobUrl { layer_path, blob_url, resolution } => {
+			Operation::SetLayerBlobUrl { layer_path, blob_url, resolution: _ } => {
 				let layer = self.layer_mut(&layer_path).unwrap_or_else(|_| panic!("Blob URL for invalid layer with path '{:?}'", layer_path));
 
 				let LayerDataType::NodeGraphFrame(node_graph_frame) = &mut layer.data else {
 					panic!("Incorrectly trying to set the image blob URL for a layer that is not a NodeGraphFrame layer type");
 				};
 
-				node_graph_frame.blob_url = Some(blob_url);
-				node_graph_frame.dimensions = resolution.into();
+				node_graph_frame.cached_output_data = CachedOutputData::BlobURL(blob_url);
 
 				self.mark_as_dirty(&layer_path)?;
 				Some([vec![DocumentChanged, LayerChanged { path: layer_path.clone() }], update_thumbnails_upstream(&layer_path)].concat())
@@ -798,8 +787,7 @@ impl Document {
 				let layer = self.layer_mut(&path).expect("Clearing node graph image for invalid layer");
 				match &mut layer.data {
 					LayerDataType::NodeGraphFrame(node_graph) => {
-						node_graph.image_data = None;
-						node_graph.blob_url = None;
+						node_graph.cached_output_data = CachedOutputData::None;
 					}
 					e => panic!("Incorrectly trying to clear the blob URL for layer of type {}", LayerDataTypeDiscriminant::from(&*e)),
 				}
@@ -828,7 +816,7 @@ impl Document {
 			}
 			Operation::SetVectorData { path, vector_data } => {
 				if let LayerDataType::NodeGraphFrame(graph) = &mut self.layer_mut(&path)?.data {
-					graph.vector_data = Some(vector_data);
+					graph.cached_output_data = CachedOutputData::VectorPath(Box::new(vector_data));
 				}
 				Some(Vec::new())
 			}

document-legacy/src/layers/layer_info.rs

@@ -438,7 +438,7 @@ impl Layer {
 
 	pub fn as_vector_data(&self) -> Option<&VectorData> {
 		match &self.data {
-			LayerDataType::NodeGraphFrame(NodeGraphFrameLayer { vector_data: Some(vector_data), .. }) => Some(vector_data),
+			LayerDataType::NodeGraphFrame(frame) => frame.as_vector_data(),
 			_ => None,
 		}
 	}
@@ -506,7 +506,7 @@ impl Layer {
 		match &self.data {
 			LayerDataType::Shape(s) => Ok(&s.style),
 			LayerDataType::Text(t) => Ok(&t.path_style),
-			LayerDataType::NodeGraphFrame(t) => t.vector_data.as_ref().map(|vector| &vector.style).ok_or(DocumentError::NotShape),
+			LayerDataType::NodeGraphFrame(t) => t.as_vector_data().map(|vector| &vector.style).ok_or(DocumentError::NotShape),
 			_ => Err(DocumentError::NotShape),
 		}
 	}

document-legacy/src/layers/nodegraph_layer.rs

@@ -1,4 +1,3 @@
-use super::base64_serde;
 use super::layer_info::LayerData;
 use super::style::{RenderData, ViewMode};
 use crate::intersection::{intersect_quad_bez_path, intersect_quad_subpath, Quad};
@@ -11,27 +10,20 @@ use serde::{Deserialize, Serialize};
 use std::fmt::Write;
 
 #[derive(Clone, Debug, Default, PartialEq, Deserialize, Serialize)]
-pub struct NodeGraphFrameLayer {
-	// Image stored in layer after generation completes
-	pub mime: String,
+pub enum CachedOutputData {
+	#[default]
+	None,
+	BlobURL(String),
+	VectorPath(Box<VectorData>),
+}
 
+#[derive(Clone, Debug, Default, PartialEq, Deserialize, Serialize)]
+pub struct NodeGraphFrameLayer {
 	/// The document node network that this layer contains
 	pub network: graph_craft::document::NodeNetwork,
 
-	// TODO: Have the browser dispose of this blob URL when this is dropped (like when the layer is deleted)
 	#[serde(skip)]
-	pub blob_url: Option<String>,
-	#[serde(skip)]
-	pub dimensions: DVec2,
-	pub image_data: Option<ImageData>,
-	pub vector_data: Option<VectorData>,
-}
-
-#[derive(Debug, Clone, Eq, PartialEq, Deserialize, Serialize, specta::Type)]
-pub struct ImageData {
-	#[serde(serialize_with = "base64_serde::as_base64", deserialize_with = "base64_serde::from_base64")]
-	#[specta(type = String)]
-	pub image_data: std::sync::Arc<Vec<u8>>,
+	pub cached_output_data: CachedOutputData,
 }
 
 impl LayerData for NodeGraphFrameLayer {
@@ -59,7 +51,8 @@ impl LayerData for NodeGraphFrameLayer {
 			.fold(String::new(), |val, (i, entry)| val + &(entry.to_string() + if i == 5 { "" } else { "," }));
 
 		// Render any paths if they exist
-		if let Some(vector_data) = &self.vector_data {
+		match &self.cached_output_data {
+			CachedOutputData::VectorPath(vector_data) => {
 				let layer_bounds = vector_data.bounding_box().unwrap_or_default();
 				let transfomed_bounds = vector_data.bounding_box_with_transform(transform).unwrap_or_default();
 
@@ -71,7 +64,8 @@ impl LayerData for NodeGraphFrameLayer {
 
 				svg.push_str(&vector_data.style.render(render_data.view_mode, svg_defs, transform, layer_bounds, transfomed_bounds));
 				let _ = write!(svg, "/>");
-		} else if let Some(blob_url) = &self.blob_url {
+			}
+			CachedOutputData::BlobURL(blob_url) => {
 				// Render the image if it exists
 				let _ = write!(
 					svg,
@@ -81,7 +75,8 @@ impl LayerData for NodeGraphFrameLayer {
 					blob_url,
 					matrix
 				);
-		} else {
+			}
+			_ => {
 				// Render a dotted blue outline if there is no image or vector data
 				let _ = write!(
 					svg,
@@ -91,6 +86,7 @@ impl LayerData for NodeGraphFrameLayer {
 					matrix,
 				);
 			}
+		}
 
 		let _ = svg.write_str(r#"</g>"#);
 
@@ -98,7 +94,7 @@ impl LayerData for NodeGraphFrameLayer {
 	}
 
 	fn bounding_box(&self, transform: glam::DAffine2, _render_data: &RenderData) -> Option<[DVec2; 2]> {
-		if let Some(vector_data) = &self.vector_data {
+		if let CachedOutputData::VectorPath(vector_data) = &self.cached_output_data {
 			return vector_data.bounding_box_with_transform(transform);
 		}
 
@@ -114,7 +110,7 @@ impl LayerData for NodeGraphFrameLayer {
 	}
 
 	fn intersects_quad(&self, quad: Quad, path: &mut Vec<LayerId>, intersections: &mut Vec<Vec<LayerId>>, _render_data: &RenderData) {
-		if let Some(vector_data) = &self.vector_data {
+		if let CachedOutputData::VectorPath(vector_data) = &self.cached_output_data {
 			let filled_style = vector_data.style.fill().is_some();
 			if vector_data.subpaths.iter().any(|subpath| intersect_quad_subpath(quad, subpath, filled_style || subpath.closed())) {
 				intersections.push(path.clone());
@@ -137,6 +133,21 @@ impl NodeGraphFrameLayer {
 	fn bounds(&self) -> BezPath {
 		kurbo::Rect::from_origin_size(kurbo::Point::ZERO, kurbo::Size::new(1., 1.)).to_path(0.)
 	}
+
+	pub fn as_vector_data(&self) -> Option<&VectorData> {
+		if let CachedOutputData::VectorPath(vector_data) = &self.cached_output_data {
+			Some(vector_data)
+		} else {
+			None
+		}
+	}
+	pub fn as_blob_url(&self) -> Option<&String> {
+		if let CachedOutputData::BlobURL(blob_url) = &self.cached_output_data {
+			Some(blob_url)
+		} else {
+			None
+		}
+	}
 }
 
 fn glam_to_kurbo(transform: DAffine2) -> Affine {

document-legacy/src/operation.rs

@@ -51,10 +51,6 @@ pub enum Operation {
 		transform: [f64; 6],
 		network: graph_craft::document::NodeNetwork,
 	},
-	SetNodeGraphFrameImageData {
-		layer_path: Vec<LayerId>,
-		image_data: Vec<u8>,
-	},
 	/// Sets a blob URL as the image source for an Image or Imaginate layer type.
 	/// **Be sure to call `FrontendMessage::TriggerRevokeBlobUrl` together with this.**
 	SetLayerBlobUrl {

editor/src/messages/portfolio/document/document_message_handler.rs

@@ -3,7 +3,7 @@ use super::utility_types::misc::DocumentRenderMode;
 use crate::application::generate_uuid;
 use crate::consts::{ASYMPTOTIC_EFFECT, DEFAULT_DOCUMENT_NAME, FILE_SAVE_SUFFIX, GRAPHITE_DOCUMENT_VERSION, SCALE_EFFECT, SCROLLBAR_SPACING, VIEWPORT_ZOOM_TO_FIT_PADDING_SCALE_FACTOR};
 use crate::messages::frontend::utility_types::ExportBounds;
-use crate::messages::frontend::utility_types::{FileType, FrontendImageData};
+use crate::messages::frontend::utility_types::FileType;
 use crate::messages::input_mapper::utility_types::macros::action_keys;
 use crate::messages::layout::utility_types::layout_widget::{Layout, LayoutGroup, Widget, WidgetCallback, WidgetHolder, WidgetLayout};
 use crate::messages::layout::utility_types::misc::LayoutTarget;
@@ -28,6 +28,7 @@ use document_legacy::document::Document as DocumentLegacy;
 use document_legacy::layers::blend_mode::BlendMode;
 use document_legacy::layers::folder_layer::FolderLayer;
 use document_legacy::layers::layer_info::{LayerDataType, LayerDataTypeDiscriminant};
+use document_legacy::layers::nodegraph_layer::CachedOutputData;
 use document_legacy::layers::style::{Fill, RenderData, ViewMode};
 use document_legacy::layers::text_layer::Font;
 use document_legacy::{DocumentError, DocumentResponse, LayerId, Operation as DocumentOperation};
@@ -422,7 +423,7 @@ impl MessageHandler<DocumentMessage, (u64, &InputPreprocessorMessageHandler, &Pe
 
 				let layer = self.document_legacy.layer(layer_path).expect("Clearing NodeGraphFrame image for invalid layer");
 				let previous_blob_url = match &layer.data {
-					LayerDataType::NodeGraphFrame(node_graph_frame) => &node_graph_frame.blob_url,
+					LayerDataType::NodeGraphFrame(node_graph_frame) => node_graph_frame.as_blob_url(),
 					x => panic!("Cannot find blob url for layer type {}", LayerDataTypeDiscriminant::from(x)),
 				};
 
@@ -839,7 +840,7 @@ impl MessageHandler<DocumentMessage, (u64, &InputPreprocessorMessageHandler, &Pe
 				// Revoke the old blob URL
 				match &layer.data {
 					LayerDataType::NodeGraphFrame(node_graph_frame) => {
-						if let Some(url) = &node_graph_frame.blob_url {
+						if let Some(url) = node_graph_frame.as_blob_url() {
 							responses.push_back(FrontendMessage::TriggerRevokeBlobUrl { url: url.clone() }.into());
 						}
 					}
@@ -1610,12 +1611,12 @@ impl DocumentMessageHandler {
 
 	/// Loads layer resources such as creating the blob URLs for the images and loading all of the fonts in the document
 	pub fn load_layer_resources(&self, responses: &mut VecDeque<Message>, root: &LayerDataType, mut path: Vec<LayerId>, document_id: u64) {
-		fn walk_layers(data: &LayerDataType, path: &mut Vec<LayerId>, image_data: &mut Vec<FrontendImageData>, fonts: &mut HashSet<Font>) {
+		fn walk_layers(data: &LayerDataType, path: &mut Vec<LayerId>, responses: &mut VecDeque<Message>, fonts: &mut HashSet<Font>) {
 			match data {
 				LayerDataType::Folder(folder) => {
 					for (id, layer) in folder.layer_ids.iter().zip(folder.layers().iter()) {
 						path.push(*id);
-						walk_layers(&layer.data, path, image_data, fonts);
+						walk_layers(&layer.data, path, responses, fonts);
 						path.pop();
 					}
 				}
@@ -1623,25 +1624,16 @@ impl DocumentMessageHandler {
 					fonts.insert(text.font.clone());
 				}
 				LayerDataType::NodeGraphFrame(node_graph_frame) => {
-					if let Some(data) = &node_graph_frame.image_data {
-						image_data.push(FrontendImageData {
-							path: path.clone(),
-							image_data: data.image_data.clone(),
-							mime: node_graph_frame.mime.clone(),
-							transform: None,
-						});
+					if node_graph_frame.cached_output_data == CachedOutputData::None {
+						responses.add(DocumentMessage::NodeGraphFrameGenerate { layer_path: path.clone() });
 					}
 				}
 				_ => {}
 			}
 		}
 
-		let mut image_data = Vec::new();
 		let mut fonts = HashSet::new();
-		walk_layers(root, &mut path, &mut image_data, &mut fonts);
-		if !image_data.is_empty() {
-			responses.push_front(FrontendMessage::UpdateImageData { document_id, image_data }.into());
-		}
+		walk_layers(root, &mut path, responses, &mut fonts);
 		for font in fonts {
 			responses.push_front(FrontendMessage::TriggerFontLoad { font, is_default: false }.into());
 		}

editor/src/messages/portfolio/document/node_graph/graph_operation_message_handler/transform_utils.rs

@@ -54,7 +54,7 @@ impl LayerBounds {
 		let layer = document.layer(layer_path).ok();
 		let bounds = layer
 			.and_then(|layer| layer.as_graph_frame().ok())
-			.and_then(|frame| frame.vector_data.as_ref().map(|vector| vector.nonzero_bounding_box()))
+			.and_then(|frame| frame.as_vector_data().as_ref().map(|vector| vector.nonzero_bounding_box()))
 			.unwrap_or([DVec2::ZERO, DVec2::ONE]);
 		let bounds_transform = DAffine2::IDENTITY;
 		let layer_transform = document.multiply_transforms(layer_path).unwrap_or_default();

editor/src/messages/tool/common_functionality/path_outline.rs

@@ -4,7 +4,6 @@ use crate::messages::prelude::*;
 
 use document_legacy::intersection::Quad;
 use document_legacy::layers::layer_info::LayerDataType;
-use document_legacy::layers::nodegraph_layer::NodeGraphFrameLayer;
 use document_legacy::layers::style::{self, Fill, RenderData, Stroke};
 use document_legacy::{LayerId, Operation};
 use graphene_std::vector::subpath::Subpath;
@@ -36,7 +35,7 @@ impl PathOutline {
 		let subpath = match &document_layer.data {
 			LayerDataType::Shape(layer_shape) => Some(layer_shape.shape.clone()),
 			LayerDataType::Text(text) => Some(text.to_subpath_nonmut(render_data)),
-			LayerDataType::NodeGraphFrame(NodeGraphFrameLayer { vector_data: Some(vector_data), .. }) => Some(Subpath::from_bezier_crate(&vector_data.subpaths)),
+			LayerDataType::NodeGraphFrame(frame) => frame.as_vector_data().map(|vector_data| Subpath::from_bezier_crate(&vector_data.subpaths)),
 			_ => document_layer.aabb_for_transform(DAffine2::IDENTITY, render_data).map(|[p1, p2]| Subpath::new_rect(p1, p2)),
 		}?;
 

editor/src/messages/tool/tool_messages/select_tool.rs

@@ -1230,7 +1230,7 @@ fn edit_layer_deepest_manipulation(intersect: &Layer, responses: &mut VecDeque<M
 		LayerDataType::Shape(_) => {
 			responses.push_front(ToolMessage::ActivateTool { tool_type: ToolType::Path }.into());
 		}
-		LayerDataType::NodeGraphFrame(frame) if frame.vector_data.is_some() => {
+		LayerDataType::NodeGraphFrame(frame) if frame.as_vector_data().is_some() => {
 			responses.push_front(ToolMessage::ActivateTool { tool_type: ToolType::Path }.into());
 		}
 		_ => {}

editor/src/node_graph_executor.rs

@@ -284,13 +284,6 @@ impl NodeGraphExecutor {
 				// Update the image data
 				let (image_data, _size) = Self::encode_img(image, None, image::ImageOutputFormat::Bmp)?;
 
-				responses.push_back(
-					Operation::SetNodeGraphFrameImageData {
-						layer_path: layer_path.clone(),
-						image_data: image_data.clone(),
-					}
-					.into(),
-				);
 				let mime = "image/bmp".to_string();
 				let image_data = std::sync::Arc::new(image_data);
 				let image_data = vec![FrontendImageData {

node-graph/gcore/Cargo.toml

@@ -12,7 +12,7 @@ license = "MIT OR Apache-2.0"
 std = ["dyn-any", "dyn-any/std", "alloc", "glam/std", "specta"]
 default = ["async", "serde", "kurbo", "log", "std"]
 log = ["dep:log"]
-serde = ["dep:serde", "glam/serde", "bezier-rs/serde"]
+serde = ["dep:serde", "glam/serde", "bezier-rs/serde", "base64"]
 gpu = ["spirv-std", "bytemuck", "glam/bytemuck", "dyn-any", "glam/libm"]
 async = ["async-trait", "alloc"]
 nightly = []
@@ -43,6 +43,7 @@ glam = { version = "^0.22", default-features = false, features = [
 	"scalar-math",
 ] }
 node-macro = { path = "../node-macro" }
+base64 = { version = "0.13", optional = true }
 specta.workspace = true
 specta.optional = true
 once_cell = { version = "1.17.0", default-features = false, optional = true }

node-graph/gcore/src/raster.rs

@@ -330,11 +330,47 @@ mod image {
 	use dyn_any::{DynAny, StaticType};
 	use glam::{DAffine2, DVec2};
 
+	#[cfg(feature = "serde")]
+	mod base64_serde {
+		//! Basic wrapper for [`serde`] for [`base64`] encoding
+
+		use crate::Color;
+		use serde::{Deserialize, Deserializer, Serializer};
+
+		pub fn as_base64<S>(key: &[Color], serializer: S) -> Result<S::Ok, S::Error>
+		where
+			S: Serializer,
+		{
+			let u8_data = key
+				.iter()
+				.flat_map(|color| [color.r(), color.g(), color.b(), color.a()].into_iter().map(|channel| (channel * 255.).clamp(0., 255.) as u8))
+				.collect::<Vec<_>>();
+			serializer.serialize_str(&base64::encode(u8_data))
+		}
+
+		pub fn from_base64<'a, D>(deserializer: D) -> Result<Vec<Color>, D::Error>
+		where
+			D: Deserializer<'a>,
+		{
+			use serde::de::Error;
+
+			let color_from_chunk = |chunk: &[u8]| Color::from_rgba8(chunk[0], chunk[1], chunk[2], chunk[3]);
+
+			let colors_from_bytes = |bytes: Vec<u8>| bytes.chunks_exact(4).map(color_from_chunk).collect();
+
+			String::deserialize(deserializer)
+				.and_then(|string| base64::decode(string).map_err(|err| Error::custom(err.to_string())))
+				.map(colors_from_bytes)
+				.map_err(serde::de::Error::custom)
+		}
+	}
+
 	#[derive(Clone, Debug, PartialEq, DynAny, Default, specta::Type)]
 	#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 	pub struct Image {
 		pub width: u32,
 		pub height: u32,
+		#[cfg_attr(feature = "serde", serde(serialize_with = "base64_serde::as_base64", deserialize_with = "base64_serde::from_base64"))]
 		pub data: Vec<Color>,
 	}