Fix the Imaginate node from crashing (#1512)

* Allow generic node input for type inference

* Make imaginate resolution picking depend on the image resolution instead of the transform

* Remove dead code

* Fix console spam after crash

* Fix crash when disconnecting Imaginate node input

* Update Imaginate tool tooltip

---------

Co-authored-by: Dennis Kobert <dennis@kobert.dev>

editor/src/application.rs

@@ -96,12 +96,8 @@ r#"
 			responses.push(res);
 		}
 		let responses = responses.pop().unwrap();
-		let trigger_message = responses[responses.len() - 2].clone();
-		if let FrontendMessage::TriggerRasterizeRegionBelowLayer { size, .. } = trigger_message {
-			assert!(size.x > 0. && size.y > 0.);
-		} else {
-			panic!();
-		}
+		// let trigger_message = responses[responses.len() - 2].clone();
+
 		println!("responses: {responses:#?}");
 	}
 }

editor/src/messages/frontend/frontend_message.rs

@@ -5,7 +5,6 @@ use crate::messages::portfolio::document::utility_types::layer_panel::{JsRawBuff
 use crate::messages::prelude::*;
 use crate::messages::tool::utility_types::HintData;
 
-use document_legacy::LayerId;
 use graph_craft::document::NodeId;
 use graphene_core::raster::color::Color;
 use graphene_core::text::Font;
@@ -92,14 +91,6 @@ pub enum FrontendMessage {
 	TriggerLoadPreferences,
 	TriggerOpenDocument,
 	TriggerPaste,
-	TriggerRasterizeRegionBelowLayer {
-		#[serde(rename = "documentId")]
-		document_id: u64,
-		#[serde(rename = "layerPath")]
-		layer_path: Vec<LayerId>,
-		svg: String,
-		size: glam::DVec2,
-	},
 	TriggerRefreshBoundsOfViewports,
 	TriggerRevokeBlobUrl {
 		url: String,

editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/document_node_types.rs

@@ -130,7 +130,7 @@ fn monitor_node() -> DocumentNode {
 		name: "Monitor".to_string(),
 		inputs: Vec::new(),
 		implementation: DocumentNodeImplementation::proto("graphene_core::memo::MonitorNode<_, _, _>"),
-		manual_composition: Some(concrete!(Footprint)),
+		manual_composition: Some(generic!(T)),
 		skip_deduplication: true,
 		..Default::default()
 	}

editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/node_properties.rs

@@ -10,6 +10,7 @@ use graph_craft::concrete;
 use graph_craft::document::value::TaggedValue;
 use graph_craft::document::{DocumentNode, NodeId, NodeInput};
 use graph_craft::imaginate_input::{ImaginateMaskStartingFill, ImaginateSamplingMethod, ImaginateServerStatus, ImaginateStatus};
+use graphene_core::memo::IORecord;
 use graphene_core::raster::{BlendMode, Color, ImageFrame, LuminanceCalculation, NoiseType, RedGreenBlue, RelativeAbsolute, SelectiveColorChoice};
 use graphene_core::text::Font;
 use graphene_core::vector::style::{FillType, GradientType, LineCap, LineJoin};
@@ -1476,6 +1477,15 @@ pub fn imaginate_properties(document_node: &DocumentNode, node_id: NodeId, conte
 		.executor
 		.introspect_node_in_network(context.network, &imaginate_node, |network| network.inputs.first().copied(), |frame: &ImageFrame<Color>| frame.transform)
 		.unwrap_or_default();
+	let image_size = context
+		.executor
+		.introspect_node_in_network(
+			context.network,
+			&imaginate_node,
+			|network| network.inputs.first().copied(),
+			|frame: &IORecord<(), ImageFrame<Color>>| (frame.output.image.width, frame.output.image.height),
+		)
+		.unwrap_or_default();
 
 	let resolution = {
 		use graphene_std::imaginate::pick_safe_imaginate_resolution;
@@ -1493,7 +1503,7 @@ pub fn imaginate_properties(document_node: &DocumentNode, node_id: NodeId, conte
 		} = &document_node.inputs[resolution_index]
 		{
 			let dimensions_is_auto = vec2.is_none();
-			let vec2 = vec2.unwrap_or_else(|| round([transform.matrix2.x_axis, transform.matrix2.y_axis].map(DVec2::length).into()));
+			let vec2 = vec2.unwrap_or_else(|| round((image_size.0 as f64, image_size.1 as f64).into()));
 
 			let layer_path = context.layer_path.to_vec();
 			widgets.extend_from_slice(&[

editor/src/messages/tool/utility_types.rs

@@ -397,7 +397,9 @@ fn list_tools_in_groups() -> Vec<Vec<ToolAvailability>> {
 		vec![
 			// Raster tool group
 			// ToolAvailability::Available(Box::<imaginate_tool::ImaginateTool>::default()), // TODO: Fix and reenable ASAP
-			ToolAvailability::ComingSoon(ToolEntry::new(ToolType::Heal, "RasterImaginateTool").tooltip("Coming Soon: Imaginate Tool - Temporarily Disabled Until Fixed (Early December 2023)")),
+			ToolAvailability::ComingSoon(
+				ToolEntry::new(ToolType::Heal, "RasterImaginateTool").tooltip("Coming Soon: Imaginate Tool - Temporarily disabled, please use Imaginate node directly from graph"),
+			),
 			ToolAvailability::Available(Box::<brush_tool::BrushTool>::default()),
 			ToolAvailability::ComingSoon(ToolEntry::new(ToolType::Heal, "RasterHealTool").tooltip("Coming Soon: Heal Tool (J)")),
 			ToolAvailability::ComingSoon(ToolEntry::new(ToolType::Clone, "RasterCloneTool").tooltip("Coming Soon: Clone Tool (C)")),

editor/src/node_graph_executor.rs

@@ -447,7 +447,10 @@ impl NodeGraphExecutor {
 		};
 		let introspection_node = find_node(wrapped_network)?;
 		let introspection = self.introspect_node(&[node_path, &[introspection_node]].concat())?;
-		let downcasted: &T = <dyn std::any::Any>::downcast_ref(introspection.as_ref())?;
+		let Some(downcasted): Option<&T> = <dyn std::any::Any>::downcast_ref(introspection.as_ref()) else {
+			log::warn!("Failed to downcast type for introspection");
+			return None;
+		};
 		Some(extract_data(downcasted))
 	}
 

frontend/src/state-providers/portfolio.ts

@@ -4,7 +4,7 @@ import { writable } from "svelte/store";
 
 import { copyToClipboardFileURL } from "@graphite/io-managers/clipboard";
 import { downloadFileText, downloadFileBlob, upload } from "@graphite/utility-functions/files";
-import { extractPixelData, imageToPNG, rasterizeSVG, rasterizeSVGCanvas } from "@graphite/utility-functions/rasterization";
+import { extractPixelData, imageToPNG, rasterizeSVG } from "@graphite/utility-functions/rasterization";
 import { type Editor } from "@graphite/wasm-communication/editor";
 import {
 	type FrontendDocumentDetails,
@@ -15,7 +15,6 @@ import {
 	TriggerDownloadTextFile,
 	TriggerImport,
 	TriggerOpenDocument,
-	TriggerRasterizeRegionBelowLayer,
 	TriggerRevokeBlobUrl,
 	UpdateActiveDocument,
 	UpdateImageData,
@@ -116,23 +115,6 @@ export function createPortfolioState(editor: Editor) {
 			editor.instance.setImageBlobURL(updateImageData.documentId, element.path, element.nodeId, blobURL, image.naturalWidth, image.naturalHeight, element.transform);
 		});
 	});
-	editor.subscriptions.subscribeJsMessage(TriggerRasterizeRegionBelowLayer, async (triggerRasterizeRegionBelowLayer) => {
-		const { documentId, layerPath, svg, size } = triggerRasterizeRegionBelowLayer;
-
-		// Rasterize the SVG to an image file
-		try {
-			if (size[0] >= 1 && size[1] >= 1) {
-				const imageData = (await rasterizeSVGCanvas(svg, size[0], size[1])).getContext("2d")?.getImageData(0, 0, size[0], size[1]);
-				if (!imageData) return;
-
-				editor.instance.renderGraphUsingRasterizedRegionBelowLayer(documentId, layerPath, new Uint8Array(imageData.data), imageData.width, imageData.height);
-			}
-		} catch (e) {
-			// getImageData may throw an exception if the resolution is too high
-			// eslint-disable-next-line no-console
-			console.error("Failed to rasterize the SVG canvas in JS to be sent back to Rust:", e);
-		}
-	});
 	editor.subscriptions.subscribeJsMessage(TriggerRevokeBlobUrl, async (triggerRevokeBlobUrl) => {
 		URL.revokeObjectURL(triggerRevokeBlobUrl.url);
 	});

frontend/src/wasm-communication/messages.ts

@@ -560,16 +560,6 @@ export class TriggerDownloadTextFile extends JsMessage {
 	readonly name!: string;
 }
 
-export class TriggerRasterizeRegionBelowLayer extends JsMessage {
-	readonly documentId!: bigint;
-
-	readonly layerPath!: BigUint64Array;
-
-	readonly svg!: string;
-
-	readonly size!: [number, number];
-}
-
 export class TriggerRefreshBoundsOfViewports extends JsMessage {}
 
 export class TriggerRevokeBlobUrl extends JsMessage {
@@ -672,8 +662,8 @@ export class DisplayEditableTextboxTransform extends JsMessage {
 export class UpdateImageData extends JsMessage {
 	readonly documentId!: bigint;
 
-	@Type(() => ImaginateImageData)
-	readonly imageData!: ImaginateImageData[];
+	@Type(() => RenderedImageData)
+	readonly imageData!: RenderedImageData[];
 }
 
 export class DisplayRemoveEditableTextbox extends JsMessage {}
@@ -710,7 +700,7 @@ export class LayerMetadata {
 
 export type LayerType = "Folder" | "Layer" | "Artboard";
 
-export class ImaginateImageData {
+export class RenderedImageData {
 	readonly path!: BigUint64Array;
 
 	readonly nodeId!: bigint;
@@ -1415,7 +1405,6 @@ export const messageMakers: Record<string, MessageMaker> = {
 	TriggerLoadPreferences,
 	TriggerOpenDocument,
 	TriggerPaste,
-	TriggerRasterizeRegionBelowLayer,
 	TriggerRefreshBoundsOfViewports,
 	TriggerRevokeBlobUrl,
 	TriggerSavePreferences,

frontend/wasm/src/editor_api.rs

@@ -2,8 +2,8 @@
 //! It serves as a thin wrapper over the editor backend API that relies
 //! on the dispatcher messaging system and more complex Rust data types.
 
-use crate::helpers::{translate_key, Error};
-use crate::{EDITOR_HAS_CRASHED, EDITOR_INSTANCES, JS_EDITOR_HANDLES};
+use crate::helpers::translate_key;
+use crate::{Error, EDITOR_HAS_CRASHED, EDITOR_INSTANCES, JS_EDITOR_HANDLES};
 
 use document_legacy::document_metadata::LayerNodeIdentifier;
 use document_legacy::LayerId;
@@ -596,13 +596,6 @@ impl JsEditorHandle {
 		}
 	}
 
-	/// Sends the blob URL generated by JS to the Imaginate layer in the respective document
-	#[wasm_bindgen(js_name = renderGraphUsingRasterizedRegionBelowLayer)]
-	pub fn render_graph_using_rasterized_region_below_layer(&self, document_id: u64, layer_path: Vec<LayerId>, _input_image_data: Vec<u8>, _width: u32, _height: u32) {
-		let message = PortfolioMessage::SubmitGraphRender { document_id, layer_path };
-		self.dispatch(message);
-	}
-
 	/// Notifies the backend that the user connected a node's primary output to one of another node's inputs
 	#[wasm_bindgen(js_name = connectNodesByLink)]
 	pub fn connect_nodes_by_link(&self, output_node: u64, output_node_connector_index: usize, input_node: u64, input_node_connector_index: usize) {

frontend/wasm/src/helpers.rs

@@ -1,68 +1,4 @@
-use crate::JS_EDITOR_HANDLES;
-
 use editor::messages::input_mapper::utility_types::input_keyboard::Key;
-use editor::messages::prelude::*;
-
-use std::panic;
-use wasm_bindgen::prelude::*;
-
-/// When a panic occurs, notify the user and log the error to the JS console before the backend dies
-pub fn panic_hook(info: &panic::PanicInfo) {
-	error!("{info}");
-
-	JS_EDITOR_HANDLES.with(|instances| {
-		instances
-			.borrow_mut()
-			.values_mut()
-			.for_each(|instance| instance.send_frontend_message_to_js_rust_proxy(FrontendMessage::DisplayDialogPanic { panic_info: info.to_string() }))
-	});
-}
-
-/// The JavaScript `Error` type
-#[wasm_bindgen]
-extern "C" {
-	#[derive(Clone, Debug)]
-	pub type Error;
-
-	#[wasm_bindgen(constructor)]
-	pub fn new(msg: &str) -> Error;
-}
-
-/// Logging to the JS console
-#[wasm_bindgen]
-extern "C" {
-	#[wasm_bindgen(js_namespace = console)]
-	fn log(msg: &str, format: &str);
-	#[wasm_bindgen(js_namespace = console)]
-	fn info(msg: &str, format: &str);
-	#[wasm_bindgen(js_namespace = console)]
-	fn warn(msg: &str, format: &str);
-	#[wasm_bindgen(js_namespace = console)]
-	fn error(msg: &str, format: &str);
-}
-
-#[derive(Default)]
-pub struct WasmLog;
-
-impl log::Log for WasmLog {
-	fn enabled(&self, metadata: &log::Metadata) -> bool {
-		metadata.level() <= log::Level::Info
-	}
-
-	fn log(&self, record: &log::Record) {
-		let (log, name, color): (fn(&str, &str), &str, &str) = match record.level() {
-			log::Level::Trace => (log, "trace", "color:plum"),
-			log::Level::Debug => (log, "debug", "color:cyan"),
-			log::Level::Warn => (warn, "warn", "color:goldenrod"),
-			log::Level::Info => (info, "info", "color:mediumseagreen"),
-			log::Level::Error => (error, "error", "color:red"),
-		};
-		let msg = &format!("%c{}\t{}", name, record.args());
-		log(msg, color)
-	}
-
-	fn flush(&self) {}
-}
 
 /// Translate a keyboard key from its JS name to its Rust `Key` enum
 pub fn translate_key(name: &str) -> Key {

frontend/wasm/src/lib.rs

@@ -7,12 +7,12 @@ extern crate log;
 pub mod editor_api;
 pub mod helpers;
 
-use helpers::{panic_hook, WasmLog};
+use editor::messages::prelude::*;
 
 use std::cell::RefCell;
 use std::collections::HashMap;
 use std::panic;
-use std::sync::atomic::AtomicBool;
+use std::sync::atomic::{AtomicBool, Ordering};
 use wasm_bindgen::prelude::*;
 
 // Set up the persistent editor backend state
@@ -33,3 +33,63 @@ pub fn init_graphite() {
 	log::set_logger(&LOGGER).expect("Failed to set logger");
 	log::set_max_level(log::LevelFilter::Debug);
 }
+
+/// When a panic occurs, notify the user and log the error to the JS console before the backend dies
+pub fn panic_hook(info: &panic::PanicInfo) {
+	EDITOR_HAS_CRASHED.store(true, Ordering::SeqCst);
+
+	error!("{info}");
+
+	JS_EDITOR_HANDLES.with(|instances| {
+		instances
+			.borrow_mut()
+			.values_mut()
+			.for_each(|instance| instance.send_frontend_message_to_js_rust_proxy(FrontendMessage::DisplayDialogPanic { panic_info: info.to_string() }))
+	});
+}
+
+/// The JavaScript `Error` type
+#[wasm_bindgen]
+extern "C" {
+	#[derive(Clone, Debug)]
+	pub type Error;
+
+	#[wasm_bindgen(constructor)]
+	pub fn new(msg: &str) -> Error;
+}
+
+/// Logging to the JS console
+#[wasm_bindgen]
+extern "C" {
+	#[wasm_bindgen(js_namespace = console)]
+	fn log(msg: &str, format: &str);
+	#[wasm_bindgen(js_namespace = console)]
+	fn info(msg: &str, format: &str);
+	#[wasm_bindgen(js_namespace = console)]
+	fn warn(msg: &str, format: &str);
+	#[wasm_bindgen(js_namespace = console)]
+	fn error(msg: &str, format: &str);
+}
+
+#[derive(Default)]
+pub struct WasmLog;
+
+impl log::Log for WasmLog {
+	fn enabled(&self, metadata: &log::Metadata) -> bool {
+		metadata.level() <= log::Level::Info
+	}
+
+	fn log(&self, record: &log::Record) {
+		let (log, name, color): (fn(&str, &str), &str, &str) = match record.level() {
+			log::Level::Trace => (log, "trace", "color:plum"),
+			log::Level::Debug => (log, "debug", "color:cyan"),
+			log::Level::Warn => (warn, "warn", "color:goldenrod"),
+			log::Level::Info => (info, "info", "color:mediumseagreen"),
+			log::Level::Error => (error, "error", "color:red"),
+		};
+		let msg = &format!("%c{}\t{}", name, record.args());
+		log(msg, color)
+	}
+
+	fn flush(&self) {}
+}

node-graph/gcore/src/graphic_element/renderer.rs

@@ -21,6 +21,10 @@ pub struct ClickTarget {
 impl ClickTarget {
 	/// Does the click target intersect the rectangle
 	pub fn intersect_rectangle(&self, document_quad: Quad, layer_transform: DAffine2) -> bool {
+		// Check if the matrix is not invertible
+		if layer_transform.matrix2.determinant() <= std::f64::EPSILON {
+			return false;
+		}
 		let quad = layer_transform.inverse() * document_quad;
 
 		// Check if outlines intersect

node-graph/gcore/src/memo.rs

@@ -45,7 +45,7 @@ impl<T, CachedNode> MemoNode<T, CachedNode> {
 	}
 }
 
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct IORecord<I, O> {
 	pub input: I,
 	pub output: O,

node-graph/graph-craft/src/proto.rs

@@ -623,9 +623,6 @@ impl TypingContext {
 			.get(&node.identifier)
 			.ok_or(format!("No implementations found for:\n\n{:?}\n\nOther implementations found:\n\n{:?}", node.identifier, self.lookup))?;
 
-		if matches!(input, Type::Generic(_)) {
-			return Err(format!("Generic types are not supported as inputs yet {:?} occurred in {:?}", input, node.identifier));
-		}
 		if parameters.iter().any(|p| {
 			matches!(p,
 			Type::Fn(_, b) if matches!(b.as_ref(), Type::Generic(_)))
@@ -636,8 +633,9 @@ impl TypingContext {
 			match (from, to) {
 				(Type::Concrete(t1), Type::Concrete(t2)) => t1 == t2,
 				(Type::Fn(a1, b1), Type::Fn(a2, b2)) => covariant(a1, a2) && covariant(b1, b2),
-				// TODO: relax this requirement when allowing generic types as inputs
-				(Type::Generic(_), _) => false,
+				// TODO: Add proper generic counting which is not based on the name
+				(Type::Generic(_), Type::Generic(_)) => true,
+				(Type::Generic(_), _) => true,
 				(_, Type::Generic(_)) => true,
 				_ => false,
 			}

node-graph/interpreted-executor/src/node_registry.rs

@@ -329,6 +329,7 @@ fn node_registry() -> HashMap<ProtoNodeIdentifier, HashMap<NodeIOTypes, NodeCons
 		)],
 		register_node!(graphene_std::raster::EmptyImageNode<_, _>, input: DAffine2, params: [Color]),
 		async_node!(graphene_core::memo::MonitorNode<_, _, _>, input: Footprint, output: ImageFrame<Color>, fn_params: [Footprint => ImageFrame<Color>]),
+		async_node!(graphene_core::memo::MonitorNode<_, _, _>, input: (), output: ImageFrame<Color>, params: [ImageFrame<Color>]),
 		async_node!(graphene_core::memo::MonitorNode<_, _, _>, input: Footprint, output: VectorData, fn_params: [Footprint => VectorData]),
 		async_node!(graphene_core::memo::MonitorNode<_, _, _>, input: Footprint, output: graphene_core::GraphicGroup, fn_params: [Footprint => graphene_core::GraphicGroup]),
 		async_node!(graphene_core::memo::MonitorNode<_, _, _>, input: Footprint, output: graphene_core::GraphicElement, fn_params: [Footprint => graphene_core::GraphicElement]),