From 833f41bccb07dfb65e26be7de3d210ebf63ae4fb Mon Sep 17 00:00:00 2001 From: Dennis Kobert Date: Wed, 13 Sep 2023 17:02:35 +0200 Subject: [PATCH] Insert pasted images as layers in document graph (#1418) Changes: Graph is evaluated on every viewport change move all navigation logic into the viewport reduce the number of js roundtrips add canvas rendering enable image pasting various cleanups Fix cache nodes being reset every evaluation --- .../document/document_message_handler.rs | 79 +++---------------- .../navigation/navigation_message_handler.rs | 25 +++--- .../node_graph/graph_operation_message.rs | 7 +- .../graph_operation_message_handler.rs | 26 +++++- .../node_graph/node_graph_message_handler.rs | 4 +- .../document_node_types.rs | 42 +--------- .../messages/portfolio/portfolio_message.rs | 10 +-- .../portfolio/portfolio_message_handler.rs | 35 ++++---- .../graph_modification_utils.rs | 10 +++ .../messages/tool/tool_messages/path_tool.rs | 6 +- editor/src/node_graph_executor.rs | 65 ++++++++------- .../src/components/panels/Document.svelte | 4 +- frontend/wasm/src/editor_api.rs | 9 +-- .../gcore/src/graphic_element/renderer.rs | 8 +- node-graph/gcore/src/types.rs | 3 +- node-graph/gstd/Cargo.toml | 1 + node-graph/gstd/src/raster.rs | 53 +++++++------ node-graph/gstd/src/wasm_application_io.rs | 36 ++++++++- .../src/dynamic_executor.rs | 3 - .../interpreted-executor/src/node_registry.rs | 2 +- 20 files changed, 202 insertions(+), 226 deletions(-) diff --git a/editor/src/messages/portfolio/document/document_message_handler.rs b/editor/src/messages/portfolio/document/document_message_handler.rs index 484e6cead..59974dd6a 100644 --- a/editor/src/messages/portfolio/document/document_message_handler.rs +++ b/editor/src/messages/portfolio/document/document_message_handler.rs @@ -6,7 +6,6 @@ use crate::messages::frontend::utility_types::ExportBounds; use crate::messages::frontend::utility_types::FileType; use crate::messages::input_mapper::utility_types::macros::action_keys; use crate::messages::layout::utility_types::widget_prelude::*; -use crate::messages::portfolio::document::node_graph::new_raster_network; use crate::messages::portfolio::document::node_graph::NodeGraphHandlerData; use crate::messages::portfolio::document::properties_panel::utility_types::PropertiesPanelMessageHandlerData; use crate::messages::portfolio::document::utility_types::clipboards::Clipboard; @@ -473,11 +472,7 @@ impl MessageHandler responses.add(InputFrameRasterizeRegionBelowLayer { layer_path }), - ImaginateGenerate { layer_path } => { - if let Some(message) = self.rasterize_region_below_layer(document_id, layer_path, preferences, persistent_data) { - responses.add(message); - } - } + ImaginateGenerate { layer_path } => responses.add(PortfolioMessage::SubmitGraphRender { document_id, layer_path }), ImaginateRandom { layer_path, imaginate_node, @@ -500,13 +495,7 @@ impl MessageHandler { - if layer_path.is_empty() { - responses.add(NodeGraphMessage::RunDocumentGraph); - } else if let Some(message) = self.rasterize_region_below_layer(document_id, layer_path, preferences, persistent_data) { - responses.add(message); - } - } + InputFrameRasterizeRegionBelowLayer { layer_path } => responses.add(PortfolioMessage::SubmitGraphRender { document_id, layer_path }), LayerChanged { affected_layer_path } => { if let Ok(layer_entry) = self.layer_panel_entry(affected_layer_path.clone(), &render_data) { responses.add(FrontendMessage::UpdateDocumentLayerDetails { data: layer_entry }); @@ -589,15 +578,10 @@ impl MessageHandler { self.layer_metadata.insert(layer_path, layer_metadata); @@ -982,45 +964,6 @@ impl DocumentMessageHandler { pub fn network(&self) -> &NodeNetwork { &self.document_legacy.document_network } - pub fn rasterize_region_below_layer(&mut self, document_id: u64, layer_path: Vec, _preferences: &PreferencesMessageHandler, persistent_data: &PersistentData) -> Option { - // Prepare the node graph input image - - let Some(node_network) = self.document_legacy.layer(&layer_path).ok().and_then(|layer| layer.as_layer_network().ok()) else { - return None; - }; - - // Check if we use the "Input Frame" node. - // TODO: Remove once rasterization is moved into a node. - let input_frame_node_id = node_network.nodes.iter().find(|(_, node)| node.name == "Input Frame").map(|(&id, _)| id); - let input_frame_connected_to_graph_output = input_frame_node_id.map_or(false, |target_node_id| node_network.connected_to_output(target_node_id)); - - // If the Input Frame node is connected upstream, rasterize the artwork below this layer by calling into JS - let response = if input_frame_connected_to_graph_output { - let old_artwork_transform = self.remove_document_transform(); - - // Calculate the size of the region to be exported and generate an SVG of the artwork below this layer within that region - let transform = self.document_legacy.multiply_transforms(&layer_path).unwrap(); - let size = DVec2::new(transform.transform_vector2(DVec2::new(1., 0.)).length(), transform.transform_vector2(DVec2::new(0., 1.)).length()); - // TODO: Test if this would be better to have a transparent background - let svg = self.render_document(size, transform.inverse(), false, persistent_data, DocumentRenderMode::OnlyBelowLayerInFolder(&layer_path)); - - self.restore_document_transform(old_artwork_transform); - - // Once JS asynchronously rasterizes the SVG, it will call the `PortfolioMessage::RenderGraphUsingRasterizedRegionBelowLayer` message with the rasterized image data - FrontendMessage::TriggerRasterizeRegionBelowLayer { document_id, layer_path, svg, size }.into() - } - // Skip taking a round trip through JS since there's nothing to rasterize, and instead directly call the message which would otherwise be called asynchronously from JS - else { - PortfolioMessage::RenderGraphUsingRasterizedRegionBelowLayer { - document_id, - layer_path, - input_image_data: vec![], - size: (0, 0), - } - .into() - }; - Some(response) - } /// Remove the artwork and artboard pan/tilt/zoom to render it without the user's viewport navigation, and save it to be restored at the end pub(crate) fn remove_document_transform(&mut self) -> DAffine2 { diff --git a/editor/src/messages/portfolio/document/navigation/navigation_message_handler.rs b/editor/src/messages/portfolio/document/navigation/navigation_message_handler.rs index f13cb97e6..a252e2b19 100644 --- a/editor/src/messages/portfolio/document/navigation/navigation_message_handler.rs +++ b/editor/src/messages/portfolio/document/navigation/navigation_message_handler.rs @@ -4,7 +4,7 @@ use crate::consts::{ }; use crate::messages::frontend::utility_types::MouseCursorIcon; use crate::messages::input_mapper::utility_types::input_keyboard::{Key, KeysGroup, MouseMotion}; -use crate::messages::input_mapper::utility_types::input_mouse::{ViewportBounds, ViewportPosition}; +use crate::messages::input_mapper::utility_types::input_mouse::ViewportPosition; use crate::messages::prelude::*; use crate::messages::tool::utility_types::{HintData, HintGroup, HintInfo}; @@ -94,7 +94,7 @@ impl MessageHandler, &InputPre responses.add(BroadcastEvent::DocumentIsDirty); responses.add(DocumentMessage::DirtyRenderDocumentInOutlineView); responses.add(PortfolioMessage::UpdateDocumentWidgets); - self.create_document_transform(&ipp.viewport_bounds, responses); + self.create_document_transform(responses); } FitViewportToSelection => { if let Some(bounds) = selection_bounds { @@ -214,7 +214,7 @@ impl MessageHandler, &InputPre } SetCanvasRotation { angle_radians } => { self.tilt = angle_radians; - self.create_document_transform(&ipp.viewport_bounds, responses); + self.create_document_transform(responses); responses.add(BroadcastEvent::DocumentIsDirty); responses.add(PortfolioMessage::UpdateDocumentWidgets); } @@ -224,7 +224,7 @@ impl MessageHandler, &InputPre responses.add(BroadcastEvent::DocumentIsDirty); responses.add(DocumentMessage::DirtyRenderDocumentInOutlineView); responses.add(PortfolioMessage::UpdateDocumentWidgets); - self.create_document_transform(&ipp.viewport_bounds, responses); + self.create_document_transform(responses); } TransformCanvasEnd { abort_transform } => { if abort_transform { @@ -235,12 +235,12 @@ impl MessageHandler, &InputPre } TransformOperation::Pan { pre_commit_pan, .. } => { self.pan = pre_commit_pan; - self.create_document_transform(&ipp.viewport_bounds, responses); + self.create_document_transform(responses); } TransformOperation::Zoom { pre_commit_zoom, .. } => { self.zoom = pre_commit_zoom; responses.add(PortfolioMessage::UpdateDocumentWidgets); - self.create_document_transform(&ipp.viewport_bounds, responses); + self.create_document_transform(responses); } } } @@ -265,7 +265,7 @@ impl MessageHandler, &InputPre self.pan += transformed_delta; responses.add(BroadcastEvent::CanvasTransformed); responses.add(BroadcastEvent::DocumentIsDirty); - self.create_document_transform(&ipp.viewport_bounds, responses); + self.create_document_transform(responses); } TranslateCanvasBegin => { responses.add(FrontendMessage::UpdateMouseCursor { cursor: MouseCursorIcon::Grabbing }); @@ -285,7 +285,7 @@ impl MessageHandler, &InputPre self.pan += transformed_delta; responses.add(BroadcastEvent::DocumentIsDirty); - self.create_document_transform(&ipp.viewport_bounds, responses); + self.create_document_transform(responses); } WheelCanvasTranslate { use_y_as_x } => { let delta = match use_y_as_x { @@ -402,11 +402,8 @@ impl NavigationMessageHandler { scale_transform * offset_transform * angle_transform * translation_transform } - fn create_document_transform(&self, viewport_bounds: &ViewportBounds, responses: &mut VecDeque) { - let half_viewport = viewport_bounds.size() / 2.; - let scaled_half_viewport = half_viewport / self.snapped_scale(); - - let transform = self.calculate_offset_transform(scaled_half_viewport); + fn create_document_transform(&self, responses: &mut VecDeque) { + let transform = self.calculate_offset_transform(DVec2::ZERO); responses.add(DocumentMessage::UpdateDocumentTransform { transform }); } @@ -414,7 +411,7 @@ impl NavigationMessageHandler { let new_viewport_bounds = viewport_bounds / zoom_factor; let delta_size = viewport_bounds - new_viewport_bounds; let mouse_fraction = mouse / viewport_bounds; - let delta = delta_size * (DVec2::splat(0.5) - mouse_fraction); + let delta = delta_size * (-mouse_fraction); NavigationMessage::TranslateCanvas { delta }.into() } diff --git a/editor/src/messages/portfolio/document/node_graph/graph_operation_message.rs b/editor/src/messages/portfolio/document/node_graph/graph_operation_message.rs index 8348ccf63..d5186b3f4 100644 --- a/editor/src/messages/portfolio/document/node_graph/graph_operation_message.rs +++ b/editor/src/messages/portfolio/document/node_graph/graph_operation_message.rs @@ -2,11 +2,12 @@ use crate::messages::prelude::*; use bezier_rs::Subpath; use graph_craft::document::NodeId; +use graphene_core::raster::ImageFrame; use graphene_core::uuid::ManipulatorGroupId; use graphene_core::vector::brush_stroke::BrushStroke; use graphene_core::vector::style::{Fill, Stroke}; use graphene_core::vector::ManipulatorPointId; -use graphene_core::Artboard; +use graphene_core::{Artboard, Color}; use glam::{DAffine2, DVec2, IVec2}; @@ -59,6 +60,10 @@ pub enum GraphOperationMessage { id: NodeId, artboard: Artboard, }, + NewBitmapLayer { + id: NodeId, + image_frame: ImageFrame, + }, NewVectorLayer { id: NodeId, subpaths: Vec>, diff --git a/editor/src/messages/portfolio/document/node_graph/graph_operation_message_handler.rs b/editor/src/messages/portfolio/document/node_graph/graph_operation_message_handler.rs index 466877c40..cfb9037a1 100644 --- a/editor/src/messages/portfolio/document/node_graph/graph_operation_message_handler.rs +++ b/editor/src/messages/portfolio/document/node_graph/graph_operation_message_handler.rs @@ -7,10 +7,11 @@ use document_legacy::document_metadata::{DocumentMetadata, LayerNodeIdentifier}; use document_legacy::{LayerId, Operation}; use graph_craft::document::value::TaggedValue; use graph_craft::document::{generate_uuid, DocumentNode, NodeId, NodeInput, NodeNetwork, NodeOutput}; +use graphene_core::raster::ImageFrame; use graphene_core::uuid::ManipulatorGroupId; use graphene_core::vector::brush_stroke::BrushStroke; use graphene_core::vector::style::{Fill, FillType, Stroke}; -use graphene_core::Artboard; +use graphene_core::{Artboard, Color}; use transform_utils::LayerBounds; use glam::{DAffine2, DVec2, IVec2}; @@ -177,6 +178,23 @@ impl<'a> ModifyInputsContext<'a> { self.responses.add(NodeGraphMessage::SendGraph { should_rerender: true }); } + fn insert_image_data(&mut self, image_frame: ImageFrame, layer: NodeId) { + let image = { + let node_type = resolve_document_node_type("Image").expect("Image node does not exist"); + node_type.to_document_node_default_inputs([Some(NodeInput::value(TaggedValue::ImageFrame(image_frame), false))], Default::default()) + }; + let sample = resolve_document_node_type("Sample").expect("Sample node does not exist").default_document_node(); + let transform = resolve_document_node_type("Transform").expect("Transform node does not exist").default_document_node(); + + let transform_id = generate_uuid(); + self.insert_node_before(transform_id, layer, 0, transform, IVec2::new(-8, 0)); + let sample_id = generate_uuid(); + self.insert_node_before(sample_id, transform_id, 0, sample, IVec2::new(-8, 0)); + let image_id = generate_uuid(); + self.insert_node_before(image_id, sample_id, 0, image, IVec2::new(-8, 0)); + self.responses.add(NodeGraphMessage::SendGraph { should_rerender: true }); + } + fn shift_upstream(&mut self, node_id: NodeId, shift: IVec2) { let mut shift_nodes = HashSet::new(); let mut stack = vec![node_id]; @@ -525,6 +543,12 @@ impl MessageHandler { + let mut modify_inputs = ModifyInputsContext::new(document, node_graph, responses); + if let Some(layer) = modify_inputs.create_layer(id, modify_inputs.network.outputs[0].node_id, 0) { + modify_inputs.insert_image_data(image_frame, layer); + } + } GraphOperationMessage::NewVectorLayer { id, subpaths } => { let mut modify_inputs = ModifyInputsContext::new(document, node_graph, responses); if let Some(layer) = modify_inputs.create_layer(id, modify_inputs.network.outputs[0].node_id, 0) { diff --git a/editor/src/messages/portfolio/document/node_graph/node_graph_message_handler.rs b/editor/src/messages/portfolio/document/node_graph/node_graph_message_handler.rs index 4d00939d3..805f9bcbf 100644 --- a/editor/src/messages/portfolio/document/node_graph/node_graph_message_handler.rs +++ b/editor/src/messages/portfolio/document/node_graph/node_graph_message_handler.rs @@ -750,11 +750,9 @@ impl<'a> MessageHandler> for NodeGrap responses.add(NodeGraphMessage::SendGraph { should_rerender: false }); } - NodeGraphMessage::RunDocumentGraph => responses.add(PortfolioMessage::RenderGraphUsingRasterizedRegionBelowLayer { + NodeGraphMessage::RunDocumentGraph => responses.add(PortfolioMessage::SubmitGraphRender { document_id: data.document_id, layer_path: Vec::new(), - input_image_data: vec![], - size: (0, 0), }), NodeGraphMessage::SelectNodes { nodes } => { self.selected_nodes = nodes; diff --git a/editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/document_node_types.rs b/editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/document_node_types.rs index cc07f5cd4..89bf77a77 100644 --- a/editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/document_node_types.rs +++ b/editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/document_node_types.rs @@ -2084,9 +2084,9 @@ fn static_nodes() -> Vec { ..Default::default() }, DocumentNodeType { - name: "Downres", + name: "Sample", category: "Structural", - identifier: NodeImplementation::proto("graphene_std::raster::DownresNode<_>"), + identifier: NodeImplementation::proto("graphene_std::raster::SampleNode<_>"), manual_composition: Some(concrete!(Footprint)), inputs: vec![DocumentInputType::value("Raseter Data", TaggedValue::ImageFrame(ImageFrame::empty()), true)], outputs: vec![DocumentOutputType::new("Vector", FrontendGraphDataType::Raster)], @@ -2517,44 +2517,6 @@ pub fn new_image_network(output_offset: i32, output_node_id: NodeId) -> NodeNetw network } -pub fn new_vector_network(subpaths: Vec>) -> NodeNetwork { - let path_generator = resolve_document_node_type("Shape").expect("Shape node does not exist"); - let cull_node = resolve_document_node_type("Cull").expect("Cull node does not exist"); - let transform = resolve_document_node_type("Transform").expect("Transform node does not exist"); - let fill = resolve_document_node_type("Fill").expect("Fill node does not exist"); - let stroke = resolve_document_node_type("Stroke").expect("Stroke node does not exist"); - let output = resolve_document_node_type("Output").expect("Output node does not exist"); - - let mut network = NodeNetwork::default(); - - network.push_node(path_generator.to_document_node_default_inputs([Some(NodeInput::value(TaggedValue::Subpaths(subpaths), false))], DocumentNodeMetadata::position((0, 4)))); - network.push_node(cull_node.to_document_node_default_inputs([None], Default::default())); - network.push_node(transform.to_document_node_default_inputs([None], Default::default())); - network.push_node(fill.to_document_node_default_inputs([None], Default::default())); - network.push_node(stroke.to_document_node_default_inputs([None], Default::default())); - network.push_node(output.to_document_node_default_inputs([None], Default::default())); - network -} - -pub fn new_raster_network(image_frame: ImageFrame) -> NodeNetwork { - let sample_node = resolve_document_node_type("Downres").expect("Downres node does not exist"); - let transform = resolve_document_node_type("Transform").expect("Transform node does not exist"); - let output = resolve_document_node_type("Output").expect("Output node does not exist"); - - let mut network = NodeNetwork::default(); - - let image_node_type = resolve_document_node_type("Image").expect("Image node should be in registry"); - - network.push_node(image_node_type.to_document_node( - [graph_craft::document::NodeInput::value(graph_craft::document::value::TaggedValue::ImageFrame(image_frame), false)], - Default::default(), - )); - network.push_node(sample_node.to_document_node_default_inputs([None], Default::default())); - network.push_node(transform.to_document_node_default_inputs([None], Default::default())); - network.push_node(output.to_document_node_default_inputs([None], Default::default())); - network -} - pub fn new_text_network(text: String, font: Font, size: f64) -> NodeNetwork { let text_generator = resolve_document_node_type("Text").expect("Text node does not exist"); let transform = resolve_document_node_type("Transform").expect("Transform node does not exist"); diff --git a/editor/src/messages/portfolio/portfolio_message.rs b/editor/src/messages/portfolio/portfolio_message.rs index af377d8e7..0acf368fd 100644 --- a/editor/src/messages/portfolio/portfolio_message.rs +++ b/editor/src/messages/portfolio/portfolio_message.rs @@ -103,12 +103,6 @@ pub enum PortfolioMessage { data: String, }, PrevDocument, - RenderGraphUsingRasterizedRegionBelowLayer { - document_id: u64, - layer_path: Vec, - input_image_data: Vec, - size: (u32, u32), - }, SelectDocument { document_id: u64, }, @@ -122,6 +116,10 @@ pub enum PortfolioMessage { blob_url: String, resolution: (f64, f64), }, + SubmitGraphRender { + document_id: u64, + layer_path: Vec, + }, UpdateDocumentWidgets, UpdateOpenDocumentsList, } diff --git a/editor/src/messages/portfolio/portfolio_message_handler.rs b/editor/src/messages/portfolio/portfolio_message_handler.rs index ccd135780..28ed93eb6 100644 --- a/editor/src/messages/portfolio/portfolio_message_handler.rs +++ b/editor/src/messages/portfolio/portfolio_message_handler.rs @@ -451,27 +451,6 @@ impl MessageHandler { - let result = self.executor.submit_node_graph_evaluation( - (document_id, &mut self.documents), - layer_path, - (input_image_data, size), - (preferences, &self.persistent_data), - responses, - ); - - if let Err(description) = result { - responses.add(DialogMessage::DisplayDialogError { - title: "Unable to update node graph".to_string(), - description, - }); - } - } PortfolioMessage::SelectDocument { document_id } => { if let Some(document) = self.active_document() { if !document.is_auto_saved() { @@ -525,6 +504,20 @@ impl MessageHandler { + let result = self.executor.submit_node_graph_evaluation( + (document_id, self.documents.get_mut(&document_id).expect("Tried to render no existent Document")), + layer_path, + ipp.viewport_bounds.size().as_uvec2(), + ); + + if let Err(description) = result { + responses.add(DialogMessage::DisplayDialogError { + title: "Unable to update node graph".to_string(), + description, + }); + } + } PortfolioMessage::UpdateDocumentWidgets => { if let Some(document) = self.active_document() { document.update_document_widgets(responses); diff --git a/editor/src/messages/tool/common_functionality/graph_modification_utils.rs b/editor/src/messages/tool/common_functionality/graph_modification_utils.rs index 36c111256..1d22b6fb9 100644 --- a/editor/src/messages/tool/common_functionality/graph_modification_utils.rs +++ b/editor/src/messages/tool/common_functionality/graph_modification_utils.rs @@ -4,8 +4,10 @@ use crate::messages::prelude::*; use bezier_rs::{ManipulatorGroup, Subpath}; use document_legacy::{document::Document, document_metadata::LayerNodeIdentifier, LayerId, Operation}; use graph_craft::document::{value::TaggedValue, DocumentNode, NodeId, NodeInput, NodeNetwork}; +use graphene_core::raster::ImageFrame; use graphene_core::uuid::ManipulatorGroupId; use graphene_core::vector::style::{FillType, Gradient}; +use graphene_core::Color; use glam::DAffine2; use std::collections::VecDeque; @@ -18,6 +20,14 @@ pub fn new_vector_layer(subpaths: Vec>, layer_path: }); } +/// Creat a new bitmap layer from an [`graphene_core::raster::ImageFrame`] +pub fn new_image_layer(image_frame: ImageFrame, layer_path: Vec, responses: &mut VecDeque) { + responses.add(GraphOperationMessage::NewBitmapLayer { + id: *layer_path.last().unwrap(), + image_frame, + }); +} + /// Create a legacy node graph frame TODO: remove pub fn new_custom_layer(network: NodeNetwork, layer_path: Vec, responses: &mut VecDeque) { responses.add(DocumentMessage::DeselectAllLayers); diff --git a/editor/src/messages/tool/tool_messages/path_tool.rs b/editor/src/messages/tool/tool_messages/path_tool.rs index 8b4dc61e1..d11c1b4cd 100644 --- a/editor/src/messages/tool/tool_messages/path_tool.rs +++ b/editor/src/messages/tool/tool_messages/path_tool.rs @@ -393,8 +393,7 @@ impl Fsm for PathToolFsmState { }; remove_bounding_box(tool_data.drag_box_overlay_layer.take(), responses); - PathToolFsmState::Ready - + PathToolFsmState::Ready } // Mouse up @@ -410,7 +409,8 @@ impl Fsm for PathToolFsmState { remove_bounding_box(tool_data.drag_box_overlay_layer.take(), responses); responses.add(PathToolMessage::SelectedPointUpdated); - } + PathToolFsmState::Ready + } (_, PathToolMessage::DragStop { shift_mirror_distance }) => { let shift_pressed = input.keyboard.get(shift_mirror_distance as usize); diff --git a/editor/src/node_graph_executor.rs b/editor/src/node_graph_executor.rs index 3f3c626b7..cd9036c1a 100644 --- a/editor/src/node_graph_executor.rs +++ b/editor/src/node_graph_executor.rs @@ -1,7 +1,6 @@ use crate::messages::frontend::utility_types::FrontendImageData; use crate::messages::portfolio::document::node_graph::wrap_network_in_scope; use crate::messages::portfolio::document::utility_types::misc::{LayerMetadata, LayerPanelEntry}; -use crate::messages::portfolio::utility_types::PersistentData; use crate::messages::prelude::*; use document_legacy::document::Document as DocumentLegacy; @@ -25,7 +24,7 @@ use graphene_core::{Color, SurfaceFrame, SurfaceId}; use graphene_std::wasm_application_io::{WasmApplicationIo, WasmEditorApi}; use interpreted_executor::dynamic_executor::DynamicExecutor; -use glam::{DAffine2, DVec2}; +use glam::{DAffine2, DVec2, UVec2}; use std::cell::RefCell; use std::rc::Rc; use std::sync::mpsc::{Receiver, Sender}; @@ -70,8 +69,8 @@ pub(crate) struct GenerationRequest { generation_id: u64, graph: NodeNetwork, path: Vec, - image_frame: Option>, transform: DAffine2, + viewport_resolution: UVec2, } pub(crate) struct GenerationResponse { @@ -139,9 +138,9 @@ impl NodeRuntime { NodeRuntimeMessage::GenerationRequest(GenerationRequest { generation_id, graph, - image_frame, transform, path, + viewport_resolution, .. }) => { let network = wrap_network_in_scope(graph); @@ -152,7 +151,7 @@ impl NodeRuntime { .map(|node| node.path.clone().unwrap_or_default()) .collect(); - let result = self.execute_network(&path, network, image_frame, transform).await; + let result = self.execute_network(&path, network, transform, viewport_resolution).await; let mut responses = VecDeque::new(); self.update_thumbnails(&path, monitor_nodes, &mut responses); let response = GenerationResponse { @@ -169,21 +168,25 @@ impl NodeRuntime { } } - async fn execute_network<'a>(&'a mut self, path: &[LayerId], scoped_network: NodeNetwork, image_frame: Option>, transform: DAffine2) -> Result { + async fn execute_network<'a>(&'a mut self, path: &[LayerId], scoped_network: NodeNetwork, transform: DAffine2, viewport_resolution: UVec2) -> Result { if self.wasm_io.is_none() { self.wasm_io = Some(WasmApplicationIo::new().await); } let editor_api = WasmEditorApi { font_cache: &self.font_cache, - image_frame, application_io: self.wasm_io.as_ref().unwrap(), node_graph_message_sender: &self.sender, imaginate_preferences: &self.imaginate_preferences, render_config: RenderConfig { - viewport: Footprint { transform, ..Default::default() }, - ..Default::default() + viewport: Footprint { + transform, + resolution: viewport_resolution, + ..Default::default() + }, + export_format: graphene_core::application_io::ExportFormat::Svg, }, + image_frame: None, }; // We assume only one output @@ -345,14 +348,14 @@ impl Default for NodeGraphExecutor { impl NodeGraphExecutor { /// Execute the network by flattening it and creating a borrow stack. - fn queue_execution(&self, network: NodeNetwork, image_frame: Option>, layer_path: Vec, transform: DAffine2) -> u64 { + fn queue_execution(&self, network: NodeNetwork, layer_path: Vec, transform: DAffine2, viewport_resolution: UVec2) -> u64 { let generation_id = generate_uuid(); let request = GenerationRequest { path: layer_path, graph: network, - image_frame, generation_id, transform, + viewport_resolution, }; self.sender.send(NodeRuntimeMessage::GenerationRequest(request)).expect("Failed to send generation request"); @@ -431,19 +434,8 @@ impl NodeGraphExecutor { } /// Evaluates a node graph, computing the entire graph - pub fn submit_node_graph_evaluation( - &mut self, - (document_id, documents): (u64, &mut HashMap), - layer_path: Vec, - (input_image_data, (width, height)): (Vec, (u32, u32)), - _persistent_data: (&PreferencesMessageHandler, &PersistentData), - _responses: &mut VecDeque, - ) -> Result<(), String> { - // Reformat the input image data into an RGBA f32 image - let image = graphene_core::raster::Image::from_image_data(&input_image_data, width, height); - + pub fn submit_node_graph_evaluation(&mut self, (document_id, document): (u64, &mut DocumentMessageHandler), layer_path: Vec, viewport_resolution: UVec2) -> Result<(), String> { // Get the node graph layer - let document = documents.get_mut(&document_id).ok_or_else(|| "Invalid document".to_string())?; let network = if layer_path.is_empty() { document.network().clone() } else { @@ -457,12 +449,10 @@ impl NodeGraphExecutor { }; // Construct the input image frame - let transform = DAffine2::IDENTITY; - let image_frame = ImageFrame { image, transform }; let document_transform = document.document_legacy.metadata.document_to_viewport; // Execute the node graph - let generation_id = self.queue_execution(network, Some(image_frame), layer_path.clone(), document_transform); + let generation_id = self.queue_execution(network, layer_path.clone(), document_transform, viewport_resolution); self.futures.insert(generation_id, ExecutionContext { layer_path, document_id }); @@ -543,13 +533,34 @@ impl NodeGraphExecutor { } TaggedValue::RenderOutput(graphene_std::wasm_application_io::RenderOutput::Svg(svg)) => { // Send to frontend - log::debug!("svg: {svg}"); + //log::debug!("svg: {svg}"); responses.add(FrontendMessage::UpdateDocumentNodeRender { svg }); responses.add(DocumentMessage::RenderScrollbars); //responses.add(FrontendMessage::UpdateDocumentNodeRender { svg }); //return Err("Graphic group (see console)".to_string()); } + TaggedValue::RenderOutput(graphene_std::wasm_application_io::RenderOutput::CanvasFrame(frame)) => { + // Send to frontend + //log::debug!("svg: {svg}"); + responses.add(DocumentMessage::RenderScrollbars); + //responses.add(FrontendMessage::UpdateDocumentNodeRender { svg }); + let matrix = frame + .transform + .to_cols_array() + .iter() + .enumerate() + .fold(String::new(), |val, (i, entry)| val + &(entry.to_string() + if i == 5 { "" } else { "," })); + let svg = format!( + r#" +
+ "#, + 1920, 1080, matrix, frame.surface_id.0 + ); + responses.add(FrontendMessage::UpdateDocumentNodeRender { svg }); + + //return Err("Graphic group (see console)".to_string()); + } TaggedValue::GraphicGroup(graphic_group) => { use graphene_core::renderer::{GraphicElementRendered, RenderParams, SvgRender}; diff --git a/frontend/src/components/panels/Document.svelte b/frontend/src/components/panels/Document.svelte index a9d72c3d8..6961bbc94 100644 --- a/frontend/src/components/panels/Document.svelte +++ b/frontend/src/components/panels/Document.svelte @@ -470,9 +470,7 @@ {@html artboardSvg} - - {@html nodeRenderSvg} - + {@html nodeRenderSvg} {@html artworkSvg} diff --git a/frontend/wasm/src/editor_api.rs b/frontend/wasm/src/editor_api.rs index 06d3f7158..935fa91bf 100644 --- a/frontend/wasm/src/editor_api.rs +++ b/frontend/wasm/src/editor_api.rs @@ -598,13 +598,8 @@ impl JsEditorHandle { /// Sends the blob URL generated by JS to the Imaginate layer in the respective document #[wasm_bindgen(js_name = renderGraphUsingRasterizedRegionBelowLayer)] - pub fn render_graph_using_rasterized_region_below_layer(&self, document_id: u64, layer_path: Vec, input_image_data: Vec, width: u32, height: u32) { - let message = PortfolioMessage::RenderGraphUsingRasterizedRegionBelowLayer { - document_id, - layer_path, - input_image_data, - size: (width, height), - }; + pub fn render_graph_using_rasterized_region_below_layer(&self, document_id: u64, layer_path: Vec, _input_image_data: Vec, _width: u32, _height: u32) { + let message = PortfolioMessage::SubmitGraphRender { document_id, layer_path }; self.dispatch(message); } diff --git a/node-graph/gcore/src/graphic_element/renderer.rs b/node-graph/gcore/src/graphic_element/renderer.rs index fe69c2686..fbb877ca4 100644 --- a/node-graph/gcore/src/graphic_element/renderer.rs +++ b/node-graph/gcore/src/graphic_element/renderer.rs @@ -294,6 +294,9 @@ impl GraphicElementRendered for ImageFrame { } ImageRenderMode::Base64 => { let image = &self.image; + if image.data.is_empty() { + return; + } let (flat_data, _, _) = image.clone().into_flat_u8(); let mut output = Vec::new(); let encoder = image::codecs::png::PngEncoder::new(&mut output); @@ -303,13 +306,12 @@ impl GraphicElementRendered for ImageFrame { let preamble = "data:image/png;base64,"; let mut base64_string = String::with_capacity(preamble.len() + output.len() * 4); base64_string.push_str(preamble); - log::debug!("len: {}", image.data.len()); base64::engine::general_purpose::STANDARD.encode_string(output, &mut base64_string); render.leaf_tag("image", |attributes| { - attributes.push("width", image.width.to_string()); + attributes.push("width", 1.to_string()); - attributes.push("height", image.height.to_string()); + attributes.push("height", 1.to_string()); attributes.push("preserveAspectRatio", "none"); attributes.push("transform", transform); attributes.push("href", base64_string) diff --git a/node-graph/gcore/src/types.rs b/node-graph/gcore/src/types.rs index b069101f2..9f05838be 100644 --- a/node-graph/gcore/src/types.rs +++ b/node-graph/gcore/src/types.rs @@ -94,7 +94,8 @@ impl PartialEq for TypeDescriptor { match (self.id, other.id) { (Some(id), Some(other_id)) => id == other_id, _ => { - warn!("TypeDescriptor::eq: comparing types without ids based on name"); + // TODO: Add a flag to disable this warning + //warn!("TypeDescriptor::eq: comparing types without ids based on name"); self.name == other.name } } diff --git a/node-graph/gstd/Cargo.toml b/node-graph/gstd/Cargo.toml index 61035bab8..37b945f18 100644 --- a/node-graph/gstd/Cargo.toml +++ b/node-graph/gstd/Cargo.toml @@ -90,5 +90,6 @@ features = [ "Navigator", "Gpu", "HtmlCanvasElement", + "HtmlImageElement", "ImageBitmapRenderingContext", ] diff --git a/node-graph/gstd/src/raster.rs b/node-graph/gstd/src/raster.rs index d84c43f26..c0ad26ed4 100644 --- a/node-graph/gstd/src/raster.rs +++ b/node-graph/gstd/src/raster.rs @@ -58,41 +58,49 @@ fn buffer_node(reader: R) -> Result, Error> { Ok(std::io::Read::bytes(reader).collect::, _>>()?) } -pub struct DownresNode { +pub struct SampleNode { image_frame: ImageFrame, } -#[node_macro::node_fn(DownresNode)] -fn downres(footprint: Footprint, image_frame: ImageFrame) -> ImageFrame { +#[node_macro::node_fn(SampleNode)] +fn sample(footprint: Footprint, image_frame: ImageFrame) -> ImageFrame { // resize the image using the image crate let image = image_frame.image; let data = bytemuck::cast_vec(image.data); let viewport_bounds = footprint.viewport_bounds_in_local_space(); - log::debug!("viewport_bounds: {viewport_bounds:?}"); - let bbox = Bbox::from_transform(image_frame.transform * DAffine2::from_scale(DVec2::new(image.width as f64, image.height as f64))); - log::debug!("local_bounds: {bbox:?}"); - let bounds = viewport_bounds.intersect(&bbox.to_axis_aligned_bbox()); - log::debug!("intersection: {bounds:?}"); - let union = viewport_bounds.union(&bbox.to_axis_aligned_bbox()); - log::debug!("union: {union:?}"); - let size = bounds.size(); + let image_bounds = Bbox::from_transform(image_frame.transform).to_axis_aligned_bbox(); + let intersection = viewport_bounds.intersect(&image_bounds); + let image_size = DAffine2::from_scale(DVec2::new(image.width as f64, image.height as f64)); + let size = intersection.size(); + let size_px = image_size.transform_vector2(size).as_uvec2(); + + // If the image would not be visible, return an empty image + if size.x <= 0. || size.y <= 0. { + return ImageFrame::empty(); + } let image_buffer = image::Rgba32FImage::from_raw(image.width, image.height, data).expect("Failed to convert internal ImageFrame into image-rs data type."); let dynamic_image: image::DynamicImage = image_buffer.into(); - let offset = (bounds.start - viewport_bounds.start).as_uvec2(); - let cropped = dynamic_image.crop_imm(offset.x, offset.y, size.x as u32, size.y as u32); + let offset = (intersection.start - image_bounds.start).max(DVec2::ZERO); + let offset_px = image_size.transform_vector2(offset).as_uvec2(); + let cropped = dynamic_image.crop_imm(offset_px.x, offset_px.y, size_px.x, size_px.y); - log::debug!("transform: {:?}", footprint.transform); - log::debug!("size: {size:?}"); let viewport_resolution_x = footprint.transform.transform_vector2(DVec2::X * size.x).length(); let viewport_resolution_y = footprint.transform.transform_vector2(DVec2::Y * size.y).length(); - let nwidth = viewport_resolution_x as u32; - let nheight = viewport_resolution_y as u32; - log::debug!("x: {viewport_resolution_x}, y: {viewport_resolution_y}"); + let mut nwidth = size_px.x; + let mut nheight = size_px.y; - let resized = cropped.resize_exact(nwidth, nheight, image::imageops::Lanczos3); + // Only downscale the image for now + let resized = if nwidth < image.width || nheight < image.height { + nwidth = viewport_resolution_x as u32; + nheight = viewport_resolution_y as u32; + // TODO: choose filter based on quality reqirements + cropped.resize_exact(nwidth, nheight, image::imageops::Triangle) + } else { + cropped + }; let buffer = resized.to_rgba32f(); let buffer = buffer.into_raw(); let vec = bytemuck::cast_vec(buffer); @@ -101,11 +109,10 @@ fn downres(footprint: Footprint, image_frame: ImageFrame) -> ImageFrame>( let mut render = SvgRender::new(); let render_params = RenderParams::new(ViewMode::Normal, graphene_core::renderer::ImageRenderMode::Base64, None, false); let output_format = editor.render_config.export_format; + let resolution = footprint.resolution; match output_format { ExportFormat::Svg => { data.render_svg(&mut render, &render_params); // TODO: reenable once we switch to full node graph - //render.format_svg((0., 0.).into(), (1., 1.).into()); + let min = footprint.transform.inverse().transform_point2((0., 0.).into()); + let max = footprint.transform.inverse().transform_point2(resolution.as_dvec2()); + render.format_svg(min, max); RenderOutput::Svg(render.svg.to_string()) } + ExportFormat::Canvas => { + data.render_svg(&mut render, &render_params); + // TODO: reenable once we switch to full node graph + let min = footprint.transform.inverse().transform_point2((0., 0.).into()); + let max = footprint.transform.inverse().transform_point2(resolution.as_dvec2()); + render.format_svg(min, max); + let string = render.svg.to_string(); + let array = string.as_bytes(); + let canvas = &surface_handle.surface; + canvas.set_width(resolution.x); + canvas.set_height(resolution.y); + + let preamble = "data:image/svg+xml;base64,"; + let mut base64_string = String::with_capacity(preamble.len() + array.len() * 4); + base64_string.push_str(preamble); + base64::engine::general_purpose::STANDARD.encode_string(array, &mut base64_string); + + let image_data = web_sys::HtmlImageElement::new().unwrap(); + image_data.set_src(base64_string.as_str()); + let context = canvas.get_context("2d").unwrap().unwrap().dyn_into::().unwrap(); + wasm_bindgen_futures::JsFuture::from(image_data.decode()).await.unwrap(); + context.draw_image_with_html_image_element(&image_data, 0.0, 0.0).unwrap(); + let frame = SurfaceHandleFrame { + surface_handle, + transform: DAffine2::IDENTITY, + }; + RenderOutput::CanvasFrame(frame.into()) + } _ => todo!("Non svg render output"), } } diff --git a/node-graph/interpreted-executor/src/dynamic_executor.rs b/node-graph/interpreted-executor/src/dynamic_executor.rs index 2bf16049a..4e5ad84cb 100644 --- a/node-graph/interpreted-executor/src/dynamic_executor.rs +++ b/node-graph/interpreted-executor/src/dynamic_executor.rs @@ -98,9 +98,6 @@ impl BorrowTree { for (id, node) in proto_network.nodes { if !self.nodes.contains_key(&id) { self.push_node(id, node, typing_context).await?; - } else { - let Some(node_container) = self.nodes.get_mut(&id) else { continue }; - node_container.reset(); } old_nodes.remove(&id); } diff --git a/node-graph/interpreted-executor/src/node_registry.rs b/node-graph/interpreted-executor/src/node_registry.rs index 66446ebbf..23def0631 100644 --- a/node-graph/interpreted-executor/src/node_registry.rs +++ b/node-graph/interpreted-executor/src/node_registry.rs @@ -732,7 +732,7 @@ fn node_registry() -> HashMap, input: Footprint, params: [VectorData]), register_node!(graphene_core::transform::CullNode<_>, input: Footprint, params: [graphene_core::Artboard]), register_node!(graphene_core::transform::CullNode<_>, input: Footprint, params: [graphene_core::GraphicGroup]), - register_node!(graphene_std::raster::DownresNode<_>, input: Footprint, params: [ImageFrame]), + register_node!(graphene_std::raster::SampleNode<_>, input: Footprint, params: [ImageFrame]), register_node!(graphene_core::vector::ResamplePoints<_>, input: VectorData, params: [f64]), register_node!(graphene_core::vector::SplineFromPointsNode, input: VectorData, params: []), register_node!(graphene_core::vector::generator_nodes::CircleGenerator<_>, input: (), params: [f32]),