Desktop: Ready runtime and render node for desktop (#2952)

* Desktop: Ready runtime and render node for desktop

* Address review comments

editor/Cargo.toml

@@ -14,7 +14,6 @@ license = "Apache-2.0"
 default = ["wasm"]
 wasm = ["wasm-bindgen", "graphene-std/wasm", "wasm-bindgen-futures"]
 gpu = ["interpreted-executor/gpu", "wgpu-executor"]
-tauri = ["ron", "decouple-execution"]
 decouple-execution = []
 resvg = ["graphene-std/resvg"]
 vello = ["graphene-std/vello", "resvg"]

editor/src/messages/layout/utility_types/widgets/input_widgets.rs

@@ -78,7 +78,9 @@ impl<'a> serde::Deserialize<'a> for CheckboxId {
 	where
 		D: serde::Deserializer<'a>,
 	{
-		let id = u64::deserialize(deserializer)?;
+		let optional_id: Option<u64> = Option::deserialize(deserializer)?;
+		// TODO: This is potentially weird because after deserialization the two labels will be decoupled if the value not existent
+		let id = optional_id.unwrap_or(0);
 		let checkbox_id = CheckboxId(OnceCell::new().into());
 		checkbox_id.0.set(id).map_err(serde::de::Error::custom)?;
 		Ok(checkbox_id)

editor/src/node_graph_executor.rs

@@ -364,6 +364,7 @@ impl NodeGraphExecutor {
 						);
 						responses.add(FrontendMessage::UpdateDocumentArtwork { svg });
 					}
+					graphene_std::wasm_application_io::RenderOutputType::Texture { .. } => {}
 					_ => {
 						return Err(format!("Invalid node graph output type: {:#?}", render_output.data));
 					}

editor/src/node_graph_executor/runtime.rs

@@ -8,7 +8,7 @@ use graph_craft::proto::GraphErrors;
 use graph_craft::wasm_application_io::EditorPreferences;
 use graph_craft::{ProtoNodeIdentifier, concrete};
 use graphene_std::Context;
-use graphene_std::application_io::{NodeGraphUpdateMessage, NodeGraphUpdateSender, RenderConfig};
+use graphene_std::application_io::{ImageTexture, NodeGraphUpdateMessage, NodeGraphUpdateSender, RenderConfig};
 use graphene_std::instances::Instance;
 use graphene_std::memo::IORecord;
 use graphene_std::renderer::{GraphicElementRendered, RenderParams, SvgRender};
@@ -16,7 +16,7 @@ use graphene_std::renderer::{RenderSvgSegmentList, SvgSegment};
 use graphene_std::text::FontCache;
 use graphene_std::vector::style::ViewMode;
 use graphene_std::vector::{VectorData, VectorDataTable};
-use graphene_std::wasm_application_io::{WasmApplicationIo, WasmEditorApi};
+use graphene_std::wasm_application_io::{RenderOutputType, WasmApplicationIo, WasmEditorApi};
 use interpreted_executor::dynamic_executor::{DynamicExecutor, IntrospectError, ResolvedDocumentNodeTypesDelta};
 use interpreted_executor::util::wrap_network_in_scope;
 use once_cell::sync::Lazy;
@@ -131,12 +131,12 @@ impl NodeRuntime {
 		}
 	}
 
-	pub async fn run(&mut self) {
+	pub async fn run(&mut self) -> Option<ImageTexture> {
 		if self.editor_api.application_io.is_none() {
 			self.editor_api = WasmEditorApi {
-				#[cfg(not(test))]
+				#[cfg(all(not(test), target_arch = "wasm32"))]
 				application_io: Some(WasmApplicationIo::new().await.into()),
-				#[cfg(test)]
+				#[cfg(any(test, not(target_arch = "wasm32")))]
 				application_io: Some(WasmApplicationIo::new_offscreen().await.into()),
 				font_cache: self.editor_api.font_cache.clone(),
 				node_graph_message_sender: Box::new(self.sender.clone()),
@@ -213,6 +213,16 @@ impl NodeRuntime {
 					// Resolve the result from the inspection by accessing the monitor node
 					let inspect_result = self.inspect_state.and_then(|state| state.access(&self.executor));
 
+					let texture = if let Ok(TaggedValue::RenderOutput(RenderOutput {
+						data: RenderOutputType::Texture(texture),
+						..
+					})) = &result
+					{
+						// We can early return becaus we know that there is at most one execution request and it will always be handled last
+						Some(texture.clone())
+					} else {
+						None
+					};
 					self.sender.send_execution_response(ExecutionResponse {
 						execution_id,
 						result,
@@ -221,9 +231,11 @@ impl NodeRuntime {
 						vector_modify: self.vector_modify.clone(),
 						inspect_result,
 					});
+					return texture;
 				}
 			}
 		}
+		None
 	}
 
 	async fn update_network(&mut self, mut graph: NodeNetwork) -> Result<ResolvedDocumentNodeTypesDelta, String> {
@@ -382,18 +394,30 @@ pub async fn introspect_node(path: &[NodeId]) -> Result<Arc<dyn std::any::Any +
 	Err(IntrospectError::RuntimeNotReady)
 }
 
-pub async fn run_node_graph() -> bool {
+pub async fn run_node_graph() -> (bool, Option<ImageTexture>) {
-	let Some(mut runtime) = NODE_RUNTIME.try_lock() else { return false };
+	let Some(mut runtime) = NODE_RUNTIME.try_lock() else { return (false, None) };
 	if let Some(ref mut runtime) = runtime.as_mut() {
-		runtime.run().await;
+		return (true, runtime.run().await);
 	}
-	true
+	(false, None)
 }
 
 pub async fn replace_node_runtime(runtime: NodeRuntime) -> Option<NodeRuntime> {
 	let mut node_runtime = NODE_RUNTIME.lock();
 	node_runtime.replace(runtime)
 }
+pub async fn replace_application_io(application_io: WasmApplicationIo) {
+	let mut node_runtime = NODE_RUNTIME.lock();
+	if let Some(node_runtime) = &mut *node_runtime {
+		node_runtime.editor_api = WasmEditorApi {
+			font_cache: node_runtime.editor_api.font_cache.clone(),
+			application_io: Some(application_io.into()),
+			node_graph_message_sender: Box::new(node_runtime.sender.clone()),
+			editor_preferences: Box::new(node_runtime.editor_preferences.clone()),
+		}
+		.into();
+	}
+}
 
 /// Which node is inspected and which monitor node is used (if any) for the current execution
 #[derive(Debug, Clone, Copy)]

editor/src/node_graph_executor/runtime_io.rs

@@ -1,24 +1,11 @@
 use super::*;
 use std::sync::mpsc::{Receiver, Sender};
-use wasm_bindgen::prelude::*;
-
-#[wasm_bindgen]
-extern "C" {
-	// Invoke with arguments (default)
-	#[wasm_bindgen(js_namespace = ["window", "__TAURI__", "core"])]
-	async fn invoke(cmd: &str, args: JsValue) -> JsValue;
-	#[wasm_bindgen(js_namespace = ["window", "__TAURI__", "core"], js_name="invoke")]
-	async fn invoke_without_arg(cmd: &str) -> JsValue;
-}
 
 /// Handles communication with the NodeRuntime, either locally or via Tauri
 #[derive(Debug)]
 pub struct NodeRuntimeIO {
 	// Send to
-	#[cfg(any(not(feature = "tauri"), test))]
 	sender: Sender<GraphRuntimeRequest>,
-	#[cfg(all(feature = "tauri", not(test)))]
-	sender: Sender<NodeGraphUpdate>,
 	receiver: Receiver<NodeGraphUpdate>,
 }
 
@@ -31,25 +18,13 @@ impl Default for NodeRuntimeIO {
 impl NodeRuntimeIO {
 	/// Creates a new NodeRuntimeIO instance
 	pub fn new() -> Self {
-		#[cfg(any(not(feature = "tauri"), test))]
+		let (response_sender, response_receiver) = std::sync::mpsc::channel();
-		{
+		let (request_sender, request_receiver) = std::sync::mpsc::channel();
-			let (response_sender, response_receiver) = std::sync::mpsc::channel();
+		futures::executor::block_on(replace_node_runtime(NodeRuntime::new(request_receiver, response_sender)));
-			let (request_sender, request_receiver) = std::sync::mpsc::channel();
-			futures::executor::block_on(replace_node_runtime(NodeRuntime::new(request_receiver, response_sender)));
 
-			Self {
+		Self {
-				sender: request_sender,
+			sender: request_sender,
-				receiver: response_receiver,
+			receiver: response_receiver,
-			}
-		}
-
-		#[cfg(all(feature = "tauri", not(test)))]
-		{
-			let (response_sender, response_receiver) = std::sync::mpsc::channel();
-			Self {
-				sender: response_sender,
-				receiver: response_receiver,
-			}
 		}
 	}
 	#[cfg(test)]
@@ -59,44 +34,11 @@ impl NodeRuntimeIO {
 
 	/// Sends a message to the NodeRuntime
 	pub fn send(&self, message: GraphRuntimeRequest) -> Result<(), String> {
-		#[cfg(any(not(feature = "tauri"), test))]
+		self.sender.send(message).map_err(|e| e.to_string())
-		{
-			self.sender.send(message).map_err(|e| e.to_string())
-		}
-
-		#[cfg(all(feature = "tauri", not(test)))]
-		{
-			let serialized = ron::to_string(&message).map_err(|e| e.to_string()).unwrap();
-			wasm_bindgen_futures::spawn_local(async move {
-				let js_message = create_message_object(&serialized);
-				invoke("runtime_message", js_message).await;
-			});
-			Ok(())
-		}
 	}
 
 	/// Receives any pending updates from the NodeRuntime
 	pub fn receive(&self) -> impl Iterator<Item = NodeGraphUpdate> + use<'_> {
-		// TODO: This introduces extra latency
-		#[cfg(all(feature = "tauri", not(test)))]
-		{
-			let sender = self.sender.clone();
-			// In the Tauri case, responses are handled separately via poll_node_runtime_updates
-			wasm_bindgen_futures::spawn_local(async move {
-				let messages = invoke_without_arg("poll_node_graph").await;
-				let vec: Vec<_> = ron::from_str(&messages.as_string().unwrap()).unwrap();
-				for message in vec {
-					sender.send(message).unwrap();
-				}
-			});
-		}
 		self.receiver.try_iter()
 	}
 }
-
-#[cfg(all(feature = "tauri", not(test)))]
-pub fn create_message_object(message: &str) -> JsValue {
-	let obj = js_sys::Object::new();
-	js_sys::Reflect::set(&obj, &JsValue::from_str("message"), &JsValue::from_str(message)).unwrap();
-	obj.into()
-}

frontend/wasm/Cargo.toml

@@ -13,7 +13,6 @@ license = "Apache-2.0"
 [features]
 default = ["gpu"]
 gpu = ["editor/gpu"]
-tauri = ["editor/tauri"]
 
 [lib]
 crate-type = ["cdylib", "rlib"]

frontend/wasm/src/editor_api.rs

@@ -931,7 +931,7 @@ async fn poll_node_graph_evaluation() {
 		return;
 	}
 
-	if !editor::node_graph_executor::run_node_graph().await {
+	if !editor::node_graph_executor::run_node_graph().await.0 {
 		return;
 	};
 

node-graph/gapplication-io/src/lib.rs

@@ -57,6 +57,15 @@ pub struct ImageTexture {
 	pub texture: (),
 }
 
+impl<'a> serde::Deserialize<'a> for ImageTexture {
+	fn deserialize<D>(_: D) -> Result<Self, D::Error>
+	where
+		D: serde::Deserializer<'a>,
+	{
+		unimplemented!("attempted to serialize a texture")
+	}
+}
+
 impl Hash for ImageTexture {
 	#[cfg(feature = "wgpu")]
 	fn hash<H: Hasher>(&self, state: &mut H) {

node-graph/graph-craft/src/document/value.rs

@@ -4,7 +4,7 @@ use crate::wasm_application_io::WasmEditorApi;
 use dyn_any::DynAny;
 pub use dyn_any::StaticType;
 pub use glam::{DAffine2, DVec2, IVec2, UVec2};
-use graphene_application_io::SurfaceFrame;
+use graphene_application_io::{ImageTexture, SurfaceFrame};
 use graphene_brush::brush_cache::BrushCache;
 use graphene_brush::brush_stroke::BrushStroke;
 use graphene_core::raster::Image;
@@ -429,7 +429,12 @@ pub struct RenderOutput {
 #[derive(Debug, Clone, Hash, PartialEq, dyn_any::DynAny, serde::Serialize, serde::Deserialize)]
 pub enum RenderOutputType {
 	CanvasFrame(SurfaceFrame),
-	Svg { svg: String, image_data: Vec<(u64, Image<Color>)> },
+	#[serde(skip)]
+	Texture(ImageTexture),
+	Svg {
+		svg: String,
+		image_data: Vec<(u64, Image<Color>)>,
+	},
 	Image(Vec<u8>),
 }
 

node-graph/graph-craft/src/wasm_application_io.rs

@@ -137,7 +137,6 @@ impl WasmApplicationIo {
 		let wgpu_available = executor.is_some();
 		WGPU_AVAILABLE.store(wgpu_available as i8, Ordering::SeqCst);
 
-		// Always enable wgpu when running with Tauri
 		let mut io = Self {
 			#[cfg(target_arch = "wasm32")]
 			ids: AtomicU64::new(0),
@@ -149,6 +148,27 @@ impl WasmApplicationIo {
 
 		io.resources.insert("null".to_string(), Arc::from(include_bytes!("null.png").to_vec()));
 
+		io
+	}
+	#[cfg(all(not(target_arch = "wasm32"), feature = "wgpu"))]
+	pub fn new_with_context(context: wgpu_executor::Context) -> Self {
+		#[cfg(feature = "wgpu")]
+		let executor = WgpuExecutor::with_context(context);
+
+		#[cfg(not(feature = "wgpu"))]
+		let wgpu_available = false;
+		#[cfg(feature = "wgpu")]
+		let wgpu_available = executor.is_some();
+		WGPU_AVAILABLE.store(wgpu_available as i8, Ordering::SeqCst);
+
+		let mut io = Self {
+			gpu_executor: executor,
+			windows: Vec::new(),
+			resources: HashMap::new(),
+		};
+
+		io.resources.insert("null".to_string(), Arc::from(include_bytes!("null.png").to_vec()));
+
 		io
 	}
 }

node-graph/gstd/src/wasm_application_io.rs

@@ -170,10 +170,10 @@ async fn render_canvas(
 	render_config: RenderConfig,
 	data: impl GraphicElementRendered,
 	editor: &WasmEditorApi,
-	surface_handle: wgpu_executor::WgpuSurface,
+	surface_handle: Option<wgpu_executor::WgpuSurface>,
 	render_params: RenderParams,
 ) -> RenderOutputType {
-	use graphene_application_io::SurfaceFrame;
+	use graphene_application_io::{ImageTexture, SurfaceFrame};
 
 	let footprint = render_config.viewport;
 	let Some(exec) = editor.application_io.as_ref().unwrap().gpu_executor() else {
@@ -194,17 +194,26 @@ async fn render_canvas(
 	if !data.contains_artboard() && !render_config.hide_artboards {
 		background = Color::WHITE;
 	}
-	exec.render_vello_scene(&scene, &surface_handle, footprint.resolution, &context, background)
+	if let Some(surface_handle) = surface_handle {
-		.await
+		exec.render_vello_scene(&scene, &surface_handle, footprint.resolution, &context, background)
-		.expect("Failed to render Vello scene");
+			.await
+			.expect("Failed to render Vello scene");
 
-	let frame = SurfaceFrame {
+		let frame = SurfaceFrame {
-		surface_id: surface_handle.window_id,
+			surface_id: surface_handle.window_id,
-		resolution: render_config.viewport.resolution,
+			resolution: render_config.viewport.resolution,
-		transform: glam::DAffine2::IDENTITY,
+			transform: glam::DAffine2::IDENTITY,
-	};
+		};
 
-	RenderOutputType::CanvasFrame(frame)
+		RenderOutputType::CanvasFrame(frame)
+	} else {
+		let texture = exec
+			.render_vello_scene_to_texture(&scene, footprint.resolution, &context, background)
+			.await
+			.expect("Failed to render Vello scene");
+
+		RenderOutputType::Texture(ImageTexture { texture: Arc::new(texture) })
+	}
 }
 
 #[cfg(target_arch = "wasm32")]
@@ -316,12 +325,14 @@ async fn render<'a: 'n, T: 'n + GraphicElementRendered + WasmNotSend>(
 	let data = data.eval(ctx.clone()).await;
 	let editor_api = editor_api.eval(None).await;
 
-	#[cfg(all(feature = "vello", not(test)))]
+	#[cfg(all(feature = "vello", not(test), target_arch = "wasm32"))]
-	let surface_handle = _surface_handle.eval(None).await;
+	let _surface_handle = _surface_handle.eval(None).await;
+	#[cfg(not(target_arch = "wasm32"))]
+	let _surface_handle: Option<wgpu_executor::WgpuSurface> = None;
 
 	let use_vello = editor_api.editor_preferences.use_vello();
-	#[cfg(all(feature = "vello", not(test)))]
+	#[cfg(all(feature = "vello", not(test), target_arch = "wasm32"))]
-	let use_vello = use_vello && surface_handle.is_some();
+	let use_vello = use_vello && _surface_handle.is_some();
 
 	let mut metadata = RenderMetadata::default();
 	data.collect_metadata(&mut metadata, footprint, None);
@@ -333,7 +344,7 @@ async fn render<'a: 'n, T: 'n + GraphicElementRendered + WasmNotSend>(
 			if use_vello && editor_api.application_io.as_ref().unwrap().gpu_executor().is_some() {
 				#[cfg(all(feature = "vello", not(test)))]
 				return RenderOutput {
-					data: render_canvas(render_config, data, editor_api, surface_handle.unwrap(), render_params).await,
+					data: render_canvas(render_config, data, editor_api, _surface_handle, render_params).await,
 					metadata,
 				};
 				#[cfg(any(not(feature = "vello"), test))]

node-graph/wgpu-executor/src/context.rs

@@ -33,7 +33,10 @@ impl Context {
 			.request_device(&wgpu::DeviceDescriptor {
 				label: None,
 				// #[cfg(not(feature = "passthrough"))]
+				#[cfg(target_arch = "wasm32")]
 				required_features: wgpu::Features::empty(),
+				#[cfg(not(target_arch = "wasm32"))]
+				required_features: wgpu::Features::PUSH_CONSTANTS,
 				// Currently disabled because not all backend support passthrough.
 				// TODO: reenable only when vulkan adapter is available
 				// #[cfg(feature = "passthrough")]
@@ -45,11 +48,6 @@ impl Context {
 			.await
 			.ok()?;
 
-		let info = adapter.get_info();
-		// skip this on LavaPipe temporarily
-		if info.vendor == 0x10005 {
-			return None;
-		}
 		Some(Self {
 			device: Arc::new(device),
 			queue: Arc::new(queue),

node-graph/wgpu-executor/src/lib.rs

@@ -141,6 +141,53 @@ impl WgpuExecutor {
 		Ok(())
 	}
 
+	pub async fn render_vello_scene_to_texture(&self, scene: &Scene, size: UVec2, context: &RenderContext, background: Color) -> Result<wgpu::Texture> {
+		let texture = self.context.device.create_texture(&wgpu::TextureDescriptor {
+			label: None,
+			size: wgpu::Extent3d {
+				width: size.x.max(1),
+				height: size.y.max(1),
+				depth_or_array_layers: 1,
+			},
+			mip_level_count: 1,
+			sample_count: 1,
+			dimension: wgpu::TextureDimension::D2,
+			usage: wgpu::TextureUsages::STORAGE_BINDING | wgpu::TextureUsages::TEXTURE_BINDING,
+			format: VELLO_SURFACE_FORMAT,
+			view_formats: &[],
+		});
+		let view = texture.create_view(&wgpu::TextureViewDescriptor::default());
+
+		let [r, g, b, _] = background.to_rgba8_srgb();
+		let render_params = RenderParams {
+			// We are using an explicit opaque color here to eliminate the alpha premultiplication step
+			// which would be required to support a transparent webgpu canvas
+			base_color: vello::peniko::Color::from_rgba8(r, g, b, 0xff),
+			width: size.x,
+			height: size.y,
+			antialiasing_method: AaConfig::Msaa16,
+		};
+
+		{
+			let mut renderer = self.vello_renderer.lock().await;
+			for (image, texture) in context.resource_overrides.iter() {
+				let texture_view = wgpu::TexelCopyTextureInfoBase {
+					texture: texture.clone(),
+					mip_level: 0,
+					origin: Origin3d::ZERO,
+					aspect: TextureAspect::All,
+				};
+				renderer.override_image(image, Some(texture_view));
+			}
+			renderer.render_to_texture(&self.context.device, &self.context.queue, scene, &view, &render_params)?;
+			for (image, _) in context.resource_overrides.iter() {
+				renderer.override_image(image, None);
+			}
+		}
+
+		Ok(texture)
+	}
+
 	#[cfg(target_arch = "wasm32")]
 	pub fn create_surface(&self, canvas: graphene_application_io::WasmSurfaceHandle) -> Result<SurfaceHandle<Surface>> {
 		let surface = self.context.instance.create_surface(wgpu::SurfaceTarget::Canvas(canvas.surface))?;
@@ -182,6 +229,24 @@ impl WgpuExecutor {
 		.map_err(|e| anyhow::anyhow!("Failed to create Vello renderer: {:?}", e))
 		.ok()?;
 
+		Some(Self {
+			context,
+			vello_renderer: vello_renderer.into(),
+		})
+	}
+	pub fn with_context(context: Context) -> Option<Self> {
+		let vello_renderer = Renderer::new(
+			&context.device,
+			RendererOptions {
+				pipeline_cache: None,
+				use_cpu: false,
+				antialiasing_support: AaSupport::all(),
+				num_init_threads: std::num::NonZeroUsize::new(1),
+			},
+		)
+		.map_err(|e| anyhow::anyhow!("Failed to create Vello renderer: {:?}", e))
+		.ok()?;
+
 		Some(Self {
 			context,
 			vello_renderer: vello_renderer.into(),