Insert pasted images as layers in document graph (#1418)

Changes:

Graph is evaluated on every viewport change
move all navigation logic into the viewport
reduce the number of js roundtrips
add canvas rendering
enable image pasting
various cleanups
Fix cache nodes being reset every evaluation

editor/src/messages/portfolio/document/document_message_handler.rs

@@ -6,7 +6,6 @@ use crate::messages::frontend::utility_types::ExportBounds;
 use crate::messages::frontend::utility_types::FileType;
 use crate::messages::input_mapper::utility_types::macros::action_keys;
 use crate::messages::layout::utility_types::widget_prelude::*;
-use crate::messages::portfolio::document::node_graph::new_raster_network;
 use crate::messages::portfolio::document::node_graph::NodeGraphHandlerData;
 use crate::messages::portfolio::document::properties_panel::utility_types::PropertiesPanelMessageHandlerData;
 use crate::messages::portfolio::document::utility_types::clipboards::Clipboard;
@@ -473,11 +472,7 @@ impl MessageHandler<DocumentMessage, (u64, &InputPreprocessorMessageHandler, &Pe
 				});
 			}
 			ImaginateClear { layer_path } => responses.add(InputFrameRasterizeRegionBelowLayer { layer_path }),
-			ImaginateGenerate { layer_path } => {
+			ImaginateGenerate { layer_path } => responses.add(PortfolioMessage::SubmitGraphRender { document_id, layer_path }),
-				if let Some(message) = self.rasterize_region_below_layer(document_id, layer_path, preferences, persistent_data) {
-					responses.add(message);
-				}
-			}
 			ImaginateRandom {
 				layer_path,
 				imaginate_node,
@@ -500,13 +495,7 @@ impl MessageHandler<DocumentMessage, (u64, &InputPreprocessorMessageHandler, &Pe
 					responses.add(DocumentMessage::ImaginateGenerate { layer_path });
 				}
 			}
-			InputFrameRasterizeRegionBelowLayer { layer_path } => {
+			InputFrameRasterizeRegionBelowLayer { layer_path } => responses.add(PortfolioMessage::SubmitGraphRender { document_id, layer_path }),
-				if layer_path.is_empty() {
-					responses.add(NodeGraphMessage::RunDocumentGraph);
-				} else if let Some(message) = self.rasterize_region_below_layer(document_id, layer_path, preferences, persistent_data) {
-					responses.add(message);
-				}
-			}
 			LayerChanged { affected_layer_path } => {
 				if let Ok(layer_entry) = self.layer_panel_entry(affected_layer_path.clone(), &render_data) {
 					responses.add(FrontendMessage::UpdateDocumentLayerDetails { data: layer_entry });
@@ -589,15 +578,10 @@ impl MessageHandler<DocumentMessage, (u64, &InputPreprocessorMessageHandler, &Pe
 
 				let image_size = DVec2::new(image.width as f64, image.height as f64);
 
-				let path = vec![generate_uuid()];
-
-				// Transform of parent folder
-				let to_parent_folder = self.document_legacy.generate_transform_across_scope(&path[..path.len() - 1], None).unwrap_or_default();
-
 				// Align the layer with the mouse or center of viewport
 				let viewport_location = mouse.map_or(ipp.viewport_bounds.center(), |pos| pos.into());
 				let center_in_viewport = DAffine2::from_translation(viewport_location - ipp.viewport_bounds.top_left);
-				let center_in_viewport_layerspace = to_parent_folder.inverse() * center_in_viewport;
+				let center_in_viewport_layerspace = center_in_viewport;
 
 				// Scale the image to fit into a 512x512 box
 				let image_size = image_size / DVec2::splat((image_size.max_element() / 512.).max(1.));
@@ -610,26 +594,23 @@ impl MessageHandler<DocumentMessage, (u64, &InputPreprocessorMessageHandler, &Pe
 				responses.add(DocumentMessage::StartTransaction);
 
 				let image_frame = ImageFrame { image, transform: DAffine2::IDENTITY };
-				let network = new_raster_network(image_frame);
 
-				responses.add(DocumentOperation::AddFrame {
+				let layer_path = self.get_path_for_new_layer();
-					path: path.clone(),
+				use crate::messages::tool::common_functionality::graph_modification_utils;
-					insert_index: -1,
+				graph_modification_utils::new_image_layer(image_frame, layer_path.clone(), responses);
-					transform: DAffine2::ZERO.to_cols_array(),
+
-					network,
-				});
 				responses.add(DocumentMessage::SetSelectedLayers {
-					replacement_selected_layers: vec![path.clone()],
+					replacement_selected_layers: vec![layer_path.clone()],
 				});
 
 				responses.add(GraphOperationMessage::TransformSet {
-					layer: path.clone(),
+					layer: layer_path.clone(),
 					transform,
 					transform_in: TransformIn::Local,
 					skip_rerender: false,
 				});
 
-				responses.add(DocumentMessage::InputFrameRasterizeRegionBelowLayer { layer_path: path });
+				responses.add(DocumentMessage::InputFrameRasterizeRegionBelowLayer { layer_path });
 
 				// Force chosen tool to be Select Tool after importing image.
 				responses.add(ToolMessage::ActivateTool { tool_type: ToolType::Select });
@@ -919,6 +900,7 @@ impl MessageHandler<DocumentMessage, (u64, &InputPreprocessorMessageHandler, &Pe
 				responses.add(FrontendMessage::UpdateDocumentTransform { transform });
 				responses.add(DocumentMessage::RenderRulers);
 				responses.add(DocumentMessage::RenderScrollbars);
+				responses.add(NodeGraphMessage::RunDocumentGraph);
 			}
 			UpdateLayerMetadata { layer_path, layer_metadata } => {
 				self.layer_metadata.insert(layer_path, layer_metadata);
@@ -982,45 +964,6 @@ impl DocumentMessageHandler {
 	pub fn network(&self) -> &NodeNetwork {
 		&self.document_legacy.document_network
 	}
-	pub fn rasterize_region_below_layer(&mut self, document_id: u64, layer_path: Vec<LayerId>, _preferences: &PreferencesMessageHandler, persistent_data: &PersistentData) -> Option<Message> {
-		// Prepare the node graph input image
-
-		let Some(node_network) = self.document_legacy.layer(&layer_path).ok().and_then(|layer| layer.as_layer_network().ok()) else {
-			return None;
-		};
-
-		// Check if we use the "Input Frame" node.
-		// TODO: Remove once rasterization is moved into a node.
-		let input_frame_node_id = node_network.nodes.iter().find(|(_, node)| node.name == "Input Frame").map(|(&id, _)| id);
-		let input_frame_connected_to_graph_output = input_frame_node_id.map_or(false, |target_node_id| node_network.connected_to_output(target_node_id));
-
-		// If the Input Frame node is connected upstream, rasterize the artwork below this layer by calling into JS
-		let response = if input_frame_connected_to_graph_output {
-			let old_artwork_transform = self.remove_document_transform();
-
-			// Calculate the size of the region to be exported and generate an SVG of the artwork below this layer within that region
-			let transform = self.document_legacy.multiply_transforms(&layer_path).unwrap();
-			let size = DVec2::new(transform.transform_vector2(DVec2::new(1., 0.)).length(), transform.transform_vector2(DVec2::new(0., 1.)).length());
-			// TODO: Test if this would be better to have a transparent background
-			let svg = self.render_document(size, transform.inverse(), false, persistent_data, DocumentRenderMode::OnlyBelowLayerInFolder(&layer_path));
-
-			self.restore_document_transform(old_artwork_transform);
-
-			// Once JS asynchronously rasterizes the SVG, it will call the `PortfolioMessage::RenderGraphUsingRasterizedRegionBelowLayer` message with the rasterized image data
-			FrontendMessage::TriggerRasterizeRegionBelowLayer { document_id, layer_path, svg, size }.into()
-		}
-		// Skip taking a round trip through JS since there's nothing to rasterize, and instead directly call the message which would otherwise be called asynchronously from JS
-		else {
-			PortfolioMessage::RenderGraphUsingRasterizedRegionBelowLayer {
-				document_id,
-				layer_path,
-				input_image_data: vec![],
-				size: (0, 0),
-			}
-			.into()
-		};
-		Some(response)
-	}
 
 	/// Remove the artwork and artboard pan/tilt/zoom to render it without the user's viewport navigation, and save it to be restored at the end
 	pub(crate) fn remove_document_transform(&mut self) -> DAffine2 {

editor/src/messages/portfolio/document/navigation/navigation_message_handler.rs

@@ -4,7 +4,7 @@ use crate::consts::{
 };
 use crate::messages::frontend::utility_types::MouseCursorIcon;
 use crate::messages::input_mapper::utility_types::input_keyboard::{Key, KeysGroup, MouseMotion};
-use crate::messages::input_mapper::utility_types::input_mouse::{ViewportBounds, ViewportPosition};
+use crate::messages::input_mapper::utility_types::input_mouse::ViewportPosition;
 use crate::messages::prelude::*;
 use crate::messages::tool::utility_types::{HintData, HintGroup, HintInfo};
 
@@ -94,7 +94,7 @@ impl MessageHandler<NavigationMessage, (&Document, Option<[DVec2; 2]>, &InputPre
 				responses.add(BroadcastEvent::DocumentIsDirty);
 				responses.add(DocumentMessage::DirtyRenderDocumentInOutlineView);
 				responses.add(PortfolioMessage::UpdateDocumentWidgets);
-				self.create_document_transform(&ipp.viewport_bounds, responses);
+				self.create_document_transform(responses);
 			}
 			FitViewportToSelection => {
 				if let Some(bounds) = selection_bounds {
@@ -214,7 +214,7 @@ impl MessageHandler<NavigationMessage, (&Document, Option<[DVec2; 2]>, &InputPre
 			}
 			SetCanvasRotation { angle_radians } => {
 				self.tilt = angle_radians;
-				self.create_document_transform(&ipp.viewport_bounds, responses);
+				self.create_document_transform(responses);
 				responses.add(BroadcastEvent::DocumentIsDirty);
 				responses.add(PortfolioMessage::UpdateDocumentWidgets);
 			}
@@ -224,7 +224,7 @@ impl MessageHandler<NavigationMessage, (&Document, Option<[DVec2; 2]>, &InputPre
 				responses.add(BroadcastEvent::DocumentIsDirty);
 				responses.add(DocumentMessage::DirtyRenderDocumentInOutlineView);
 				responses.add(PortfolioMessage::UpdateDocumentWidgets);
-				self.create_document_transform(&ipp.viewport_bounds, responses);
+				self.create_document_transform(responses);
 			}
 			TransformCanvasEnd { abort_transform } => {
 				if abort_transform {
@@ -235,12 +235,12 @@ impl MessageHandler<NavigationMessage, (&Document, Option<[DVec2; 2]>, &InputPre
 						}
 						TransformOperation::Pan { pre_commit_pan, .. } => {
 							self.pan = pre_commit_pan;
-							self.create_document_transform(&ipp.viewport_bounds, responses);
+							self.create_document_transform(responses);
 						}
 						TransformOperation::Zoom { pre_commit_zoom, .. } => {
 							self.zoom = pre_commit_zoom;
 							responses.add(PortfolioMessage::UpdateDocumentWidgets);
-							self.create_document_transform(&ipp.viewport_bounds, responses);
+							self.create_document_transform(responses);
 						}
 					}
 				}
@@ -265,7 +265,7 @@ impl MessageHandler<NavigationMessage, (&Document, Option<[DVec2; 2]>, &InputPre
 				self.pan += transformed_delta;
 				responses.add(BroadcastEvent::CanvasTransformed);
 				responses.add(BroadcastEvent::DocumentIsDirty);
-				self.create_document_transform(&ipp.viewport_bounds, responses);
+				self.create_document_transform(responses);
 			}
 			TranslateCanvasBegin => {
 				responses.add(FrontendMessage::UpdateMouseCursor { cursor: MouseCursorIcon::Grabbing });
@@ -285,7 +285,7 @@ impl MessageHandler<NavigationMessage, (&Document, Option<[DVec2; 2]>, &InputPre
 
 				self.pan += transformed_delta;
 				responses.add(BroadcastEvent::DocumentIsDirty);
-				self.create_document_transform(&ipp.viewport_bounds, responses);
+				self.create_document_transform(responses);
 			}
 			WheelCanvasTranslate { use_y_as_x } => {
 				let delta = match use_y_as_x {
@@ -402,11 +402,8 @@ impl NavigationMessageHandler {
 		scale_transform * offset_transform * angle_transform * translation_transform
 	}
 
-	fn create_document_transform(&self, viewport_bounds: &ViewportBounds, responses: &mut VecDeque<Message>) {
+	fn create_document_transform(&self, responses: &mut VecDeque<Message>) {
-		let half_viewport = viewport_bounds.size() / 2.;
+		let transform = self.calculate_offset_transform(DVec2::ZERO);
-		let scaled_half_viewport = half_viewport / self.snapped_scale();
-
-		let transform = self.calculate_offset_transform(scaled_half_viewport);
 		responses.add(DocumentMessage::UpdateDocumentTransform { transform });
 	}
 
@@ -414,7 +411,7 @@ impl NavigationMessageHandler {
 		let new_viewport_bounds = viewport_bounds / zoom_factor;
 		let delta_size = viewport_bounds - new_viewport_bounds;
 		let mouse_fraction = mouse / viewport_bounds;
-		let delta = delta_size * (DVec2::splat(0.5) - mouse_fraction);
+		let delta = delta_size * (-mouse_fraction);
 
 		NavigationMessage::TranslateCanvas { delta }.into()
 	}

editor/src/messages/portfolio/document/node_graph/graph_operation_message.rs

@@ -2,11 +2,12 @@ use crate::messages::prelude::*;
 
 use bezier_rs::Subpath;
 use graph_craft::document::NodeId;
+use graphene_core::raster::ImageFrame;
 use graphene_core::uuid::ManipulatorGroupId;
 use graphene_core::vector::brush_stroke::BrushStroke;
 use graphene_core::vector::style::{Fill, Stroke};
 use graphene_core::vector::ManipulatorPointId;
-use graphene_core::Artboard;
+use graphene_core::{Artboard, Color};
 
 use glam::{DAffine2, DVec2, IVec2};
 
@@ -59,6 +60,10 @@ pub enum GraphOperationMessage {
 		id: NodeId,
 		artboard: Artboard,
 	},
+	NewBitmapLayer {
+		id: NodeId,
+		image_frame: ImageFrame<Color>,
+	},
 	NewVectorLayer {
 		id: NodeId,
 		subpaths: Vec<Subpath<ManipulatorGroupId>>,

editor/src/messages/portfolio/document/node_graph/graph_operation_message_handler.rs

@@ -7,10 +7,11 @@ use document_legacy::document_metadata::{DocumentMetadata, LayerNodeIdentifier};
 use document_legacy::{LayerId, Operation};
 use graph_craft::document::value::TaggedValue;
 use graph_craft::document::{generate_uuid, DocumentNode, NodeId, NodeInput, NodeNetwork, NodeOutput};
+use graphene_core::raster::ImageFrame;
 use graphene_core::uuid::ManipulatorGroupId;
 use graphene_core::vector::brush_stroke::BrushStroke;
 use graphene_core::vector::style::{Fill, FillType, Stroke};
-use graphene_core::Artboard;
+use graphene_core::{Artboard, Color};
 use transform_utils::LayerBounds;
 
 use glam::{DAffine2, DVec2, IVec2};
@@ -177,6 +178,23 @@ impl<'a> ModifyInputsContext<'a> {
 		self.responses.add(NodeGraphMessage::SendGraph { should_rerender: true });
 	}
 
+	fn insert_image_data(&mut self, image_frame: ImageFrame<Color>, layer: NodeId) {
+		let image = {
+			let node_type = resolve_document_node_type("Image").expect("Image node does not exist");
+			node_type.to_document_node_default_inputs([Some(NodeInput::value(TaggedValue::ImageFrame(image_frame), false))], Default::default())
+		};
+		let sample = resolve_document_node_type("Sample").expect("Sample node does not exist").default_document_node();
+		let transform = resolve_document_node_type("Transform").expect("Transform node does not exist").default_document_node();
+
+		let transform_id = generate_uuid();
+		self.insert_node_before(transform_id, layer, 0, transform, IVec2::new(-8, 0));
+		let sample_id = generate_uuid();
+		self.insert_node_before(sample_id, transform_id, 0, sample, IVec2::new(-8, 0));
+		let image_id = generate_uuid();
+		self.insert_node_before(image_id, sample_id, 0, image, IVec2::new(-8, 0));
+		self.responses.add(NodeGraphMessage::SendGraph { should_rerender: true });
+	}
+
 	fn shift_upstream(&mut self, node_id: NodeId, shift: IVec2) {
 		let mut shift_nodes = HashSet::new();
 		let mut stack = vec![node_id];
@@ -525,6 +543,12 @@ impl MessageHandler<GraphOperationMessage, (&mut Document, &mut NodeGraphMessage
 					modify_inputs.insert_artboard(artboard, layer);
 				}
 			}
+			GraphOperationMessage::NewBitmapLayer { id, image_frame } => {
+				let mut modify_inputs = ModifyInputsContext::new(document, node_graph, responses);
+				if let Some(layer) = modify_inputs.create_layer(id, modify_inputs.network.outputs[0].node_id, 0) {
+					modify_inputs.insert_image_data(image_frame, layer);
+				}
+			}
 			GraphOperationMessage::NewVectorLayer { id, subpaths } => {
 				let mut modify_inputs = ModifyInputsContext::new(document, node_graph, responses);
 				if let Some(layer) = modify_inputs.create_layer(id, modify_inputs.network.outputs[0].node_id, 0) {

editor/src/messages/portfolio/document/node_graph/node_graph_message_handler.rs

@@ -750,11 +750,9 @@ impl<'a> MessageHandler<NodeGraphMessage, NodeGraphHandlerData<'a>> for NodeGrap
 
 				responses.add(NodeGraphMessage::SendGraph { should_rerender: false });
 			}
-			NodeGraphMessage::RunDocumentGraph => responses.add(PortfolioMessage::RenderGraphUsingRasterizedRegionBelowLayer {
+			NodeGraphMessage::RunDocumentGraph => responses.add(PortfolioMessage::SubmitGraphRender {
 				document_id: data.document_id,
 				layer_path: Vec::new(),
-				input_image_data: vec![],
-				size: (0, 0),
 			}),
 			NodeGraphMessage::SelectNodes { nodes } => {
 				self.selected_nodes = nodes;

editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/document_node_types.rs

@@ -2084,9 +2084,9 @@ fn static_nodes() -> Vec<DocumentNodeType> {
 			..Default::default()
 		},
 		DocumentNodeType {
-			name: "Downres",
+			name: "Sample",
 			category: "Structural",
-			identifier: NodeImplementation::proto("graphene_std::raster::DownresNode<_>"),
+			identifier: NodeImplementation::proto("graphene_std::raster::SampleNode<_>"),
 			manual_composition: Some(concrete!(Footprint)),
 			inputs: vec![DocumentInputType::value("Raseter Data", TaggedValue::ImageFrame(ImageFrame::empty()), true)],
 			outputs: vec![DocumentOutputType::new("Vector", FrontendGraphDataType::Raster)],
@@ -2517,44 +2517,6 @@ pub fn new_image_network(output_offset: i32, output_node_id: NodeId) -> NodeNetw
 	network
 }
 
-pub fn new_vector_network(subpaths: Vec<bezier_rs::Subpath<uuid::ManipulatorGroupId>>) -> NodeNetwork {
-	let path_generator = resolve_document_node_type("Shape").expect("Shape node does not exist");
-	let cull_node = resolve_document_node_type("Cull").expect("Cull node does not exist");
-	let transform = resolve_document_node_type("Transform").expect("Transform node does not exist");
-	let fill = resolve_document_node_type("Fill").expect("Fill node does not exist");
-	let stroke = resolve_document_node_type("Stroke").expect("Stroke node does not exist");
-	let output = resolve_document_node_type("Output").expect("Output node does not exist");
-
-	let mut network = NodeNetwork::default();
-
-	network.push_node(path_generator.to_document_node_default_inputs([Some(NodeInput::value(TaggedValue::Subpaths(subpaths), false))], DocumentNodeMetadata::position((0, 4))));
-	network.push_node(cull_node.to_document_node_default_inputs([None], Default::default()));
-	network.push_node(transform.to_document_node_default_inputs([None], Default::default()));
-	network.push_node(fill.to_document_node_default_inputs([None], Default::default()));
-	network.push_node(stroke.to_document_node_default_inputs([None], Default::default()));
-	network.push_node(output.to_document_node_default_inputs([None], Default::default()));
-	network
-}
-
-pub fn new_raster_network(image_frame: ImageFrame<Color>) -> NodeNetwork {
-	let sample_node = resolve_document_node_type("Downres").expect("Downres node does not exist");
-	let transform = resolve_document_node_type("Transform").expect("Transform node does not exist");
-	let output = resolve_document_node_type("Output").expect("Output node does not exist");
-
-	let mut network = NodeNetwork::default();
-
-	let image_node_type = resolve_document_node_type("Image").expect("Image node should be in registry");
-
-	network.push_node(image_node_type.to_document_node(
-		[graph_craft::document::NodeInput::value(graph_craft::document::value::TaggedValue::ImageFrame(image_frame), false)],
-		Default::default(),
-	));
-	network.push_node(sample_node.to_document_node_default_inputs([None], Default::default()));
-	network.push_node(transform.to_document_node_default_inputs([None], Default::default()));
-	network.push_node(output.to_document_node_default_inputs([None], Default::default()));
-	network
-}
-
 pub fn new_text_network(text: String, font: Font, size: f64) -> NodeNetwork {
 	let text_generator = resolve_document_node_type("Text").expect("Text node does not exist");
 	let transform = resolve_document_node_type("Transform").expect("Transform node does not exist");

editor/src/messages/portfolio/portfolio_message.rs

@@ -103,12 +103,6 @@ pub enum PortfolioMessage {
 		data: String,
 	},
 	PrevDocument,
-	RenderGraphUsingRasterizedRegionBelowLayer {
-		document_id: u64,
-		layer_path: Vec<LayerId>,
-		input_image_data: Vec<u8>,
-		size: (u32, u32),
-	},
 	SelectDocument {
 		document_id: u64,
 	},
@@ -122,6 +116,10 @@ pub enum PortfolioMessage {
 		blob_url: String,
 		resolution: (f64, f64),
 	},
+	SubmitGraphRender {
+		document_id: u64,
+		layer_path: Vec<LayerId>,
+	},
 	UpdateDocumentWidgets,
 	UpdateOpenDocumentsList,
 }

editor/src/messages/portfolio/portfolio_message_handler.rs

@@ -451,27 +451,6 @@ impl MessageHandler<PortfolioMessage, (&InputPreprocessorMessageHandler, &Prefer
 					responses.add(PortfolioMessage::SelectDocument { document_id: prev_id });
 				}
 			}
-			PortfolioMessage::RenderGraphUsingRasterizedRegionBelowLayer {
-				document_id,
-				layer_path,
-				input_image_data,
-				size,
-			} => {
-				let result = self.executor.submit_node_graph_evaluation(
-					(document_id, &mut self.documents),
-					layer_path,
-					(input_image_data, size),
-					(preferences, &self.persistent_data),
-					responses,
-				);
-
-				if let Err(description) = result {
-					responses.add(DialogMessage::DisplayDialogError {
-						title: "Unable to update node graph".to_string(),
-						description,
-					});
-				}
-			}
 			PortfolioMessage::SelectDocument { document_id } => {
 				if let Some(document) = self.active_document() {
 					if !document.is_auto_saved() {
@@ -525,6 +504,20 @@ impl MessageHandler<PortfolioMessage, (&InputPreprocessorMessageHandler, &Prefer
 				};
 				responses.add(PortfolioMessage::DocumentPassMessage { document_id, message });
 			}
+			PortfolioMessage::SubmitGraphRender { document_id, layer_path } => {
+				let result = self.executor.submit_node_graph_evaluation(
+					(document_id, self.documents.get_mut(&document_id).expect("Tried to render no existent Document")),
+					layer_path,
+					ipp.viewport_bounds.size().as_uvec2(),
+				);
+
+				if let Err(description) = result {
+					responses.add(DialogMessage::DisplayDialogError {
+						title: "Unable to update node graph".to_string(),
+						description,
+					});
+				}
+			}
 			PortfolioMessage::UpdateDocumentWidgets => {
 				if let Some(document) = self.active_document() {
 					document.update_document_widgets(responses);

editor/src/messages/tool/common_functionality/graph_modification_utils.rs

@@ -4,8 +4,10 @@ use crate::messages::prelude::*;
 use bezier_rs::{ManipulatorGroup, Subpath};
 use document_legacy::{document::Document, document_metadata::LayerNodeIdentifier, LayerId, Operation};
 use graph_craft::document::{value::TaggedValue, DocumentNode, NodeId, NodeInput, NodeNetwork};
+use graphene_core::raster::ImageFrame;
 use graphene_core::uuid::ManipulatorGroupId;
 use graphene_core::vector::style::{FillType, Gradient};
+use graphene_core::Color;
 
 use glam::DAffine2;
 use std::collections::VecDeque;
@@ -18,6 +20,14 @@ pub fn new_vector_layer(subpaths: Vec<Subpath<ManipulatorGroupId>>, layer_path:
 	});
 }
 
+/// Creat a new bitmap layer from an [`graphene_core::raster::ImageFrame<Color>`]
+pub fn new_image_layer(image_frame: ImageFrame<Color>, layer_path: Vec<LayerId>, responses: &mut VecDeque<Message>) {
+	responses.add(GraphOperationMessage::NewBitmapLayer {
+		id: *layer_path.last().unwrap(),
+		image_frame,
+	});
+}
+
 /// Create a legacy node graph frame TODO: remove
 pub fn new_custom_layer(network: NodeNetwork, layer_path: Vec<LayerId>, responses: &mut VecDeque<Message>) {
 	responses.add(DocumentMessage::DeselectAllLayers);

editor/src/messages/tool/tool_messages/path_tool.rs

@@ -393,8 +393,7 @@ impl Fsm for PathToolFsmState {
 				};
 				remove_bounding_box(tool_data.drag_box_overlay_layer.take(), responses);
 
-					PathToolFsmState::Ready
+				PathToolFsmState::Ready
-
 			}
 
 			// Mouse up
@@ -410,7 +409,8 @@ impl Fsm for PathToolFsmState {
 				remove_bounding_box(tool_data.drag_box_overlay_layer.take(), responses);
 
 				responses.add(PathToolMessage::SelectedPointUpdated);
-				}
+				PathToolFsmState::Ready
+			}
 
 			(_, PathToolMessage::DragStop { shift_mirror_distance }) => {
 				let shift_pressed = input.keyboard.get(shift_mirror_distance as usize);

editor/src/node_graph_executor.rs

@@ -1,7 +1,6 @@
 use crate::messages::frontend::utility_types::FrontendImageData;
 use crate::messages::portfolio::document::node_graph::wrap_network_in_scope;
 use crate::messages::portfolio::document::utility_types::misc::{LayerMetadata, LayerPanelEntry};
-use crate::messages::portfolio::utility_types::PersistentData;
 use crate::messages::prelude::*;
 
 use document_legacy::document::Document as DocumentLegacy;
@@ -25,7 +24,7 @@ use graphene_core::{Color, SurfaceFrame, SurfaceId};
 use graphene_std::wasm_application_io::{WasmApplicationIo, WasmEditorApi};
 use interpreted_executor::dynamic_executor::DynamicExecutor;
 
-use glam::{DAffine2, DVec2};
+use glam::{DAffine2, DVec2, UVec2};
 use std::cell::RefCell;
 use std::rc::Rc;
 use std::sync::mpsc::{Receiver, Sender};
@@ -70,8 +69,8 @@ pub(crate) struct GenerationRequest {
 	generation_id: u64,
 	graph: NodeNetwork,
 	path: Vec<LayerId>,
-	image_frame: Option<ImageFrame<Color>>,
 	transform: DAffine2,
+	viewport_resolution: UVec2,
 }
 
 pub(crate) struct GenerationResponse {
@@ -139,9 +138,9 @@ impl NodeRuntime {
 				NodeRuntimeMessage::GenerationRequest(GenerationRequest {
 					generation_id,
 					graph,
-					image_frame,
 					transform,
 					path,
+					viewport_resolution,
 					..
 				}) => {
 					let network = wrap_network_in_scope(graph);
@@ -152,7 +151,7 @@ impl NodeRuntime {
 						.map(|node| node.path.clone().unwrap_or_default())
 						.collect();
 
-					let result = self.execute_network(&path, network, image_frame, transform).await;
+					let result = self.execute_network(&path, network, transform, viewport_resolution).await;
 					let mut responses = VecDeque::new();
 					self.update_thumbnails(&path, monitor_nodes, &mut responses);
 					let response = GenerationResponse {
@@ -169,21 +168,25 @@ impl NodeRuntime {
 		}
 	}
 
-	async fn execute_network<'a>(&'a mut self, path: &[LayerId], scoped_network: NodeNetwork, image_frame: Option<ImageFrame<Color>>, transform: DAffine2) -> Result<TaggedValue, String> {
+	async fn execute_network<'a>(&'a mut self, path: &[LayerId], scoped_network: NodeNetwork, transform: DAffine2, viewport_resolution: UVec2) -> Result<TaggedValue, String> {
 		if self.wasm_io.is_none() {
 			self.wasm_io = Some(WasmApplicationIo::new().await);
 		}
 
 		let editor_api = WasmEditorApi {
 			font_cache: &self.font_cache,
-			image_frame,
 			application_io: self.wasm_io.as_ref().unwrap(),
 			node_graph_message_sender: &self.sender,
 			imaginate_preferences: &self.imaginate_preferences,
 			render_config: RenderConfig {
-				viewport: Footprint { transform, ..Default::default() },
+				viewport: Footprint {
-				..Default::default()
+					transform,
+					resolution: viewport_resolution,
+					..Default::default()
+				},
+				export_format: graphene_core::application_io::ExportFormat::Svg,
 			},
+			image_frame: None,
 		};
 
 		// We assume only one output
@@ -345,14 +348,14 @@ impl Default for NodeGraphExecutor {
 
 impl NodeGraphExecutor {
 	/// Execute the network by flattening it and creating a borrow stack.
-	fn queue_execution(&self, network: NodeNetwork, image_frame: Option<ImageFrame<Color>>, layer_path: Vec<LayerId>, transform: DAffine2) -> u64 {
+	fn queue_execution(&self, network: NodeNetwork, layer_path: Vec<LayerId>, transform: DAffine2, viewport_resolution: UVec2) -> u64 {
 		let generation_id = generate_uuid();
 		let request = GenerationRequest {
 			path: layer_path,
 			graph: network,
-			image_frame,
 			generation_id,
 			transform,
+			viewport_resolution,
 		};
 		self.sender.send(NodeRuntimeMessage::GenerationRequest(request)).expect("Failed to send generation request");
 
@@ -431,19 +434,8 @@ impl NodeGraphExecutor {
 	}
 
 	/// Evaluates a node graph, computing the entire graph
-	pub fn submit_node_graph_evaluation(
+	pub fn submit_node_graph_evaluation(&mut self, (document_id, document): (u64, &mut DocumentMessageHandler), layer_path: Vec<LayerId>, viewport_resolution: UVec2) -> Result<(), String> {
-		&mut self,
-		(document_id, documents): (u64, &mut HashMap<u64, DocumentMessageHandler>),
-		layer_path: Vec<LayerId>,
-		(input_image_data, (width, height)): (Vec<u8>, (u32, u32)),
-		_persistent_data: (&PreferencesMessageHandler, &PersistentData),
-		_responses: &mut VecDeque<Message>,
-	) -> Result<(), String> {
-		// Reformat the input image data into an RGBA f32 image
-		let image = graphene_core::raster::Image::from_image_data(&input_image_data, width, height);
-
 		// Get the node graph layer
-		let document = documents.get_mut(&document_id).ok_or_else(|| "Invalid document".to_string())?;
 		let network = if layer_path.is_empty() {
 			document.network().clone()
 		} else {
@@ -457,12 +449,10 @@ impl NodeGraphExecutor {
 		};
 
 		// Construct the input image frame
-		let transform = DAffine2::IDENTITY;
-		let image_frame = ImageFrame { image, transform };
 		let document_transform = document.document_legacy.metadata.document_to_viewport;
 
 		// Execute the node graph
-		let generation_id = self.queue_execution(network, Some(image_frame), layer_path.clone(), document_transform);
+		let generation_id = self.queue_execution(network, layer_path.clone(), document_transform, viewport_resolution);
 
 		self.futures.insert(generation_id, ExecutionContext { layer_path, document_id });
 
@@ -543,13 +533,34 @@ impl NodeGraphExecutor {
 			}
 			TaggedValue::RenderOutput(graphene_std::wasm_application_io::RenderOutput::Svg(svg)) => {
 				// Send to frontend
-				log::debug!("svg: {svg}");
+				//log::debug!("svg: {svg}");
 				responses.add(FrontendMessage::UpdateDocumentNodeRender { svg });
 				responses.add(DocumentMessage::RenderScrollbars);
 				//responses.add(FrontendMessage::UpdateDocumentNodeRender { svg });
 
 				//return Err("Graphic group (see console)".to_string());
 			}
+			TaggedValue::RenderOutput(graphene_std::wasm_application_io::RenderOutput::CanvasFrame(frame)) => {
+				// Send to frontend
+				//log::debug!("svg: {svg}");
+				responses.add(DocumentMessage::RenderScrollbars);
+				//responses.add(FrontendMessage::UpdateDocumentNodeRender { svg });
+				let matrix = frame
+					.transform
+					.to_cols_array()
+					.iter()
+					.enumerate()
+					.fold(String::new(), |val, (i, entry)| val + &(entry.to_string() + if i == 5 { "" } else { "," }));
+				let svg = format!(
+					r#"
+					<svg><foreignObject width="{}" height="{}" transform="matrix({})"><div data-canvas-placeholder="canvas{}"></div></foreignObject></svg>
+					"#,
+					1920, 1080, matrix, frame.surface_id.0
+				);
+				responses.add(FrontendMessage::UpdateDocumentNodeRender { svg });
+
+				//return Err("Graphic group (see console)".to_string());
+			}
 			TaggedValue::GraphicGroup(graphic_group) => {
 				use graphene_core::renderer::{GraphicElementRendered, RenderParams, SvgRender};
 

frontend/src/components/panels/Document.svelte

@@ -470,9 +470,7 @@
 							{@html artboardSvg}
 						</svg>
 						<svg class="artboards" style:width={canvasWidthCSS} style:height={canvasHeightCSS}>
-							<g id="transform-group" transform={$document.artworkTransform}>
+							{@html nodeRenderSvg}
-								{@html nodeRenderSvg}
-							</g>
 						</svg>
 						<svg class="artwork" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" style:width={canvasWidthCSS} style:height={canvasHeightCSS}>
 							{@html artworkSvg}

frontend/wasm/src/editor_api.rs

@@ -598,13 +598,8 @@ impl JsEditorHandle {
 
 	/// Sends the blob URL generated by JS to the Imaginate layer in the respective document
 	#[wasm_bindgen(js_name = renderGraphUsingRasterizedRegionBelowLayer)]
-	pub fn render_graph_using_rasterized_region_below_layer(&self, document_id: u64, layer_path: Vec<LayerId>, input_image_data: Vec<u8>, width: u32, height: u32) {
+	pub fn render_graph_using_rasterized_region_below_layer(&self, document_id: u64, layer_path: Vec<LayerId>, _input_image_data: Vec<u8>, _width: u32, _height: u32) {
-		let message = PortfolioMessage::RenderGraphUsingRasterizedRegionBelowLayer {
+		let message = PortfolioMessage::SubmitGraphRender { document_id, layer_path };
-			document_id,
-			layer_path,
-			input_image_data,
-			size: (width, height),
-		};
 		self.dispatch(message);
 	}
 

node-graph/gcore/src/graphic_element/renderer.rs

@@ -294,6 +294,9 @@ impl GraphicElementRendered for ImageFrame<Color> {
 			}
 			ImageRenderMode::Base64 => {
 				let image = &self.image;
+				if image.data.is_empty() {
+					return;
+				}
 				let (flat_data, _, _) = image.clone().into_flat_u8();
 				let mut output = Vec::new();
 				let encoder = image::codecs::png::PngEncoder::new(&mut output);
@@ -303,13 +306,12 @@ impl GraphicElementRendered for ImageFrame<Color> {
 				let preamble = "data:image/png;base64,";
 				let mut base64_string = String::with_capacity(preamble.len() + output.len() * 4);
 				base64_string.push_str(preamble);
-				log::debug!("len: {}", image.data.len());
 				base64::engine::general_purpose::STANDARD.encode_string(output, &mut base64_string);
 
 				render.leaf_tag("image", |attributes| {
-					attributes.push("width", image.width.to_string());
+					attributes.push("width", 1.to_string());
 
-					attributes.push("height", image.height.to_string());
+					attributes.push("height", 1.to_string());
 					attributes.push("preserveAspectRatio", "none");
 					attributes.push("transform", transform);
 					attributes.push("href", base64_string)

node-graph/gcore/src/types.rs

@@ -94,7 +94,8 @@ impl PartialEq for TypeDescriptor {
 		match (self.id, other.id) {
 			(Some(id), Some(other_id)) => id == other_id,
 			_ => {
-				warn!("TypeDescriptor::eq: comparing types without ids based on name");
+				// TODO: Add a flag to disable this warning
+				//warn!("TypeDescriptor::eq: comparing types without ids based on name");
 				self.name == other.name
 			}
 		}

node-graph/gstd/Cargo.toml

@@ -90,5 +90,6 @@ features = [
 	"Navigator",
 	"Gpu",
 	"HtmlCanvasElement",
+	"HtmlImageElement",
 	"ImageBitmapRenderingContext",
 ]

node-graph/gstd/src/raster.rs

@@ -58,41 +58,49 @@ fn buffer_node<R: std::io::Read>(reader: R) -> Result<Vec<u8>, Error> {
 	Ok(std::io::Read::bytes(reader).collect::<Result<Vec<_>, _>>()?)
 }
 
-pub struct DownresNode<ImageFrame> {
+pub struct SampleNode<ImageFrame> {
 	image_frame: ImageFrame,
 }
 
-#[node_macro::node_fn(DownresNode)]
+#[node_macro::node_fn(SampleNode)]
-fn downres(footprint: Footprint, image_frame: ImageFrame<Color>) -> ImageFrame<Color> {
+fn sample(footprint: Footprint, image_frame: ImageFrame<Color>) -> ImageFrame<Color> {
 	// resize the image using the image crate
 	let image = image_frame.image;
 	let data = bytemuck::cast_vec(image.data);
 
 	let viewport_bounds = footprint.viewport_bounds_in_local_space();
-	log::debug!("viewport_bounds: {viewport_bounds:?}");
+	let image_bounds = Bbox::from_transform(image_frame.transform).to_axis_aligned_bbox();
-	let bbox = Bbox::from_transform(image_frame.transform * DAffine2::from_scale(DVec2::new(image.width as f64, image.height as f64)));
+	let intersection = viewport_bounds.intersect(&image_bounds);
-	log::debug!("local_bounds: {bbox:?}");
+	let image_size = DAffine2::from_scale(DVec2::new(image.width as f64, image.height as f64));
-	let bounds = viewport_bounds.intersect(&bbox.to_axis_aligned_bbox());
+	let size = intersection.size();
-	log::debug!("intersection: {bounds:?}");
+	let size_px = image_size.transform_vector2(size).as_uvec2();
-	let union = viewport_bounds.union(&bbox.to_axis_aligned_bbox());
+
-	log::debug!("union: {union:?}");
+	// If the image would not be visible, return an empty image
-	let size = bounds.size();
+	if size.x <= 0. || size.y <= 0. {
+		return ImageFrame::empty();
+	}
 
 	let image_buffer = image::Rgba32FImage::from_raw(image.width, image.height, data).expect("Failed to convert internal ImageFrame into image-rs data type.");
 
 	let dynamic_image: image::DynamicImage = image_buffer.into();
-	let offset = (bounds.start - viewport_bounds.start).as_uvec2();
+	let offset = (intersection.start - image_bounds.start).max(DVec2::ZERO);
-	let cropped = dynamic_image.crop_imm(offset.x, offset.y, size.x as u32, size.y as u32);
+	let offset_px = image_size.transform_vector2(offset).as_uvec2();
+	let cropped = dynamic_image.crop_imm(offset_px.x, offset_px.y, size_px.x, size_px.y);
 
-	log::debug!("transform: {:?}", footprint.transform);
-	log::debug!("size: {size:?}");
 	let viewport_resolution_x = footprint.transform.transform_vector2(DVec2::X * size.x).length();
 	let viewport_resolution_y = footprint.transform.transform_vector2(DVec2::Y * size.y).length();
-	let nwidth = viewport_resolution_x as u32;
+	let mut nwidth = size_px.x;
-	let nheight = viewport_resolution_y as u32;
+	let mut nheight = size_px.y;
-	log::debug!("x: {viewport_resolution_x}, y: {viewport_resolution_y}");
 
-	let resized = cropped.resize_exact(nwidth, nheight, image::imageops::Lanczos3);
+	// Only downscale the image for now
+	let resized = if nwidth < image.width || nheight < image.height {
+		nwidth = viewport_resolution_x as u32;
+		nheight = viewport_resolution_y as u32;
+		// TODO: choose filter based on quality reqirements
+		cropped.resize_exact(nwidth, nheight, image::imageops::Triangle)
+	} else {
+		cropped
+	};
 	let buffer = resized.to_rgba32f();
 	let buffer = buffer.into_raw();
 	let vec = bytemuck::cast_vec(buffer);
@@ -101,11 +109,10 @@ fn downres(footprint: Footprint, image_frame: ImageFrame<Color>) -> ImageFrame<C
 		height: nheight,
 		data: vec,
 	};
+	// we need to adjust the offset if we truncate the offset calculation
 
-	ImageFrame {
+	let new_transform = image_frame.transform * DAffine2::from_translation(offset) * DAffine2::from_scale(DVec2::new(size.x, size.y));
-		image,
+	ImageFrame { image, transform: new_transform }
-		transform: image_frame.transform,
-	}
 }
 
 #[derive(Debug, Clone, Copy)]

node-graph/gstd/src/wasm_application_io.rs

@@ -28,6 +28,9 @@ use web_sys::{CanvasRenderingContext2d, HtmlCanvasElement};
 #[cfg(feature = "wgpu")]
 use wgpu_executor::WgpuExecutor;
 
+use base64::Engine;
+use glam::DAffine2;
+
 pub struct Canvas(CanvasRenderingContext2d);
 
 #[derive(Debug, Default)]
@@ -301,14 +304,45 @@ async fn render_node<'a: 'input, F: Future<Output = GraphicGroup>>(
 	let mut render = SvgRender::new();
 	let render_params = RenderParams::new(ViewMode::Normal, graphene_core::renderer::ImageRenderMode::Base64, None, false);
 	let output_format = editor.render_config.export_format;
+	let resolution = footprint.resolution;
 
 	match output_format {
 		ExportFormat::Svg => {
 			data.render_svg(&mut render, &render_params);
 			// TODO: reenable once we switch to full node graph
-			//render.format_svg((0., 0.).into(), (1., 1.).into());
+			let min = footprint.transform.inverse().transform_point2((0., 0.).into());
+			let max = footprint.transform.inverse().transform_point2(resolution.as_dvec2());
+			render.format_svg(min, max);
 			RenderOutput::Svg(render.svg.to_string())
 		}
+		ExportFormat::Canvas => {
+			data.render_svg(&mut render, &render_params);
+			// TODO: reenable once we switch to full node graph
+			let min = footprint.transform.inverse().transform_point2((0., 0.).into());
+			let max = footprint.transform.inverse().transform_point2(resolution.as_dvec2());
+			render.format_svg(min, max);
+			let string = render.svg.to_string();
+			let array = string.as_bytes();
+			let canvas = &surface_handle.surface;
+			canvas.set_width(resolution.x);
+			canvas.set_height(resolution.y);
+
+			let preamble = "data:image/svg+xml;base64,";
+			let mut base64_string = String::with_capacity(preamble.len() + array.len() * 4);
+			base64_string.push_str(preamble);
+			base64::engine::general_purpose::STANDARD.encode_string(array, &mut base64_string);
+
+			let image_data = web_sys::HtmlImageElement::new().unwrap();
+			image_data.set_src(base64_string.as_str());
+			let context = canvas.get_context("2d").unwrap().unwrap().dyn_into::<CanvasRenderingContext2d>().unwrap();
+			wasm_bindgen_futures::JsFuture::from(image_data.decode()).await.unwrap();
+			context.draw_image_with_html_image_element(&image_data, 0.0, 0.0).unwrap();
+			let frame = SurfaceHandleFrame {
+				surface_handle,
+				transform: DAffine2::IDENTITY,
+			};
+			RenderOutput::CanvasFrame(frame.into())
+		}
 		_ => todo!("Non svg render output"),
 	}
 }

node-graph/interpreted-executor/src/dynamic_executor.rs

@@ -98,9 +98,6 @@ impl BorrowTree {
 		for (id, node) in proto_network.nodes {
 			if !self.nodes.contains_key(&id) {
 				self.push_node(id, node, typing_context).await?;
-			} else {
-				let Some(node_container) = self.nodes.get_mut(&id) else { continue };
-				node_container.reset();
 			}
 			old_nodes.remove(&id);
 		}

node-graph/interpreted-executor/src/node_registry.rs

@@ -732,7 +732,7 @@ fn node_registry() -> HashMap<NodeIdentifier, HashMap<NodeIOTypes, NodeConstruct
 		register_node!(graphene_core::transform::CullNode<_>, input: Footprint, params: [VectorData]),
 		register_node!(graphene_core::transform::CullNode<_>, input: Footprint, params: [graphene_core::Artboard]),
 		register_node!(graphene_core::transform::CullNode<_>, input: Footprint, params: [graphene_core::GraphicGroup]),
-		register_node!(graphene_std::raster::DownresNode<_>, input: Footprint, params: [ImageFrame<Color>]),
+		register_node!(graphene_std::raster::SampleNode<_>, input: Footprint, params: [ImageFrame<Color>]),
 		register_node!(graphene_core::vector::ResamplePoints<_>, input: VectorData, params: [f64]),
 		register_node!(graphene_core::vector::SplineFromPointsNode, input: VectorData, params: []),
 		register_node!(graphene_core::vector::generator_nodes::CircleGenerator<_>, input: (), params: [f32]),