Use canvas as target for raster rendering (#1256)

* Implement ApplicationIo

* Simplify output duplication logic

* Fix let node initialization for ExtractImageFrame

* Async macros

* Use manual node registry impl

* Fix canvas insertion into the dom

node-graph/gcore/Cargo.toml

@@ -18,7 +18,7 @@ std = [
 	"num-traits/std",
 	"rustybuzz",
 ]
-default = ["async", "serde", "kurbo", "log", "std", "rand_chacha"]
+default = ["async", "serde", "kurbo", "log", "std", "rand_chacha", "wasm"]
 log = ["dep:log"]
 serde = [
 	"dep:serde",
@@ -32,6 +32,7 @@ async = ["async-trait", "alloc"]
 nightly = []
 alloc = ["dyn-any", "bezier-rs", "once_cell"]
 type_id_logging = []
+wasm = ["wasm-bindgen", "web-sys", "js-sys", "std"]
 
 [dependencies]
 dyn-any = { path = "../../libraries/dyn-any", features = [
@@ -68,3 +69,18 @@ num-derive = { version = "0.3.3" }
 num-traits = { version = "0.2.15", default-features = false, features = [
 	"i128",
 ] }
+
+
+wasm-bindgen = { version = "0.2.84", optional = true }
+js-sys = { version = "0.3.55", optional = true }
+
+[dependencies.web-sys]
+version = "0.3.4"
+optional = true
+features = [
+	"Window",
+	"CanvasRenderingContext2d",
+	"ImageData",
+	"Document",
+	"HtmlCanvasElement",
+]

node-graph/gcore/src/application_io.rs

@@ -0,0 +1,159 @@
+use crate::raster::ImageFrame;
+use crate::transform::Transform;
+use crate::transform::TransformMut;
+use crate::Color;
+use crate::Node;
+use alloc::sync::Arc;
+use dyn_any::StaticType;
+use dyn_any::StaticTypeSized;
+use glam::DAffine2;
+
+use core::hash::{Hash, Hasher};
+
+use crate::text::FontCache;
+
+use core::fmt::Debug;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
+pub struct SurfaceId(pub u64);
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
+pub struct SurfaceFrame {
+	pub surface_id: SurfaceId,
+	pub transform: DAffine2,
+}
+
+impl Hash for SurfaceFrame {
+	fn hash<H: Hasher>(&self, state: &mut H) {
+		self.surface_id.hash(state);
+		self.transform.to_cols_array().iter().for_each(|x| x.to_bits().hash(state));
+	}
+}
+
+unsafe impl StaticType for SurfaceFrame {
+	type Static = SurfaceFrame;
+}
+
+#[derive(Clone)]
+pub struct SurfaceHandle<'a, Surface> {
+	pub surface_id: SurfaceId,
+	pub surface: Surface,
+	application_io: &'a dyn ApplicationIo<Surface = Surface>,
+}
+
+unsafe impl<T: 'static> StaticType for SurfaceHandle<'_, T> {
+	type Static = SurfaceHandle<'static, T>;
+}
+
+#[derive(Clone)]
+pub struct SurfaceHandleFrame<'a, Surface> {
+	pub surface_handle: Arc<SurfaceHandle<'a, Surface>>,
+	pub transform: DAffine2,
+}
+
+unsafe impl<T: 'static> StaticType for SurfaceHandleFrame<'_, T> {
+	type Static = SurfaceHandleFrame<'static, T>;
+}
+
+impl<T> Transform for SurfaceHandleFrame<'_, T> {
+	fn transform(&self) -> DAffine2 {
+		self.transform
+	}
+}
+
+impl<T> TransformMut for SurfaceHandleFrame<'_, T> {
+	fn transform_mut(&mut self) -> &mut DAffine2 {
+		&mut self.transform
+	}
+}
+
+// TODO: think about how to automatically clean up memory
+/*
+impl<'a, Surface> Drop for SurfaceHandle<'a, Surface> {
+	fn drop(&mut self) {
+		self.application_io.destroy_surface(self.surface_id)
+	}
+}*/
+
+pub trait ApplicationIo {
+	type Surface;
+	fn create_surface(&self) -> SurfaceHandle<Self::Surface>;
+	fn destroy_surface(&self, surface_id: SurfaceId);
+}
+
+impl<T: ApplicationIo> ApplicationIo for &T {
+	type Surface = T::Surface;
+	fn create_surface(&self) -> SurfaceHandle<T::Surface> {
+		(**self).create_surface()
+	}
+
+	fn destroy_surface(&self, surface_id: SurfaceId) {
+		(**self).destroy_surface(surface_id)
+	}
+}
+
+pub struct EditorApi<'a, Io> {
+	pub image_frame: Option<ImageFrame<Color>>,
+	pub font_cache: &'a FontCache,
+	pub application_io: &'a Io,
+}
+
+impl<'a, Io> Clone for EditorApi<'a, Io> {
+	fn clone(&self) -> Self {
+		Self {
+			image_frame: self.image_frame.clone(),
+			font_cache: self.font_cache,
+			application_io: self.application_io,
+		}
+	}
+}
+
+impl<'a, T> PartialEq for EditorApi<'a, T> {
+	fn eq(&self, other: &Self) -> bool {
+		self.image_frame == other.image_frame && self.font_cache == other.font_cache
+	}
+}
+
+impl<'a, T> Hash for EditorApi<'a, T> {
+	fn hash<H: Hasher>(&self, state: &mut H) {
+		self.image_frame.hash(state);
+		self.font_cache.hash(state);
+	}
+}
+
+impl<'a, T> Debug for EditorApi<'a, T> {
+	fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+		f.debug_struct("EditorApi").field("image_frame", &self.image_frame).field("font_cache", &self.font_cache).finish()
+	}
+}
+
+unsafe impl<T: StaticTypeSized> StaticType for EditorApi<'_, T> {
+	type Static = EditorApi<'static, T::Static>;
+}
+
+impl<'a, T> AsRef<EditorApi<'a, T>> for EditorApi<'a, T> {
+	fn as_ref(&self) -> &EditorApi<'a, T> {
+		self
+	}
+}
+
+#[derive(Debug, Clone, Copy, Default)]
+pub struct ExtractImageFrame;
+
+impl<'a: 'input, 'input, T> Node<'input, &'a EditorApi<'a, T>> for ExtractImageFrame {
+	type Output = ImageFrame<Color>;
+	fn eval(&'input self, editor_api: &'a EditorApi<'a, T>) -> Self::Output {
+		editor_api.image_frame.clone().unwrap_or(ImageFrame::identity())
+	}
+}
+
+impl ExtractImageFrame {
+	pub fn new() -> Self {
+		Self
+	}
+}
+
+#[cfg(feature = "wasm")]
+pub mod wasm_application_io;

node-graph/gcore/src/application_io/wasm_application_io.rs

@@ -0,0 +1,131 @@
+use std::{cell::RefCell, collections::HashMap, sync::Mutex};
+
+use super::{ApplicationIo, SurfaceHandle, SurfaceHandleFrame, SurfaceId};
+use crate::{
+	raster::{color::SRGBA8, ImageFrame, Pixel},
+	Node,
+};
+use alloc::sync::Arc;
+use dyn_any::StaticType;
+use js_sys::{Object, Reflect};
+use wasm_bindgen::{Clamped, JsCast, JsValue};
+use web_sys::{window, CanvasRenderingContext2d, HtmlCanvasElement};
+
+pub struct Canvas(CanvasRenderingContext2d);
+
+#[derive(Debug, Default)]
+pub struct WasmApplicationIo {
+	ids: RefCell<u64>,
+	canvases: RefCell<HashMap<SurfaceId, CanvasRenderingContext2d>>,
+}
+
+impl WasmApplicationIo {
+	pub fn new() -> Self {
+		Self::default()
+	}
+}
+
+unsafe impl StaticType for WasmApplicationIo {
+	type Static = WasmApplicationIo;
+}
+
+pub type WasmEditorApi<'a> = super::EditorApi<'a, WasmApplicationIo>;
+
+impl ApplicationIo for WasmApplicationIo {
+	type Surface = CanvasRenderingContext2d;
+
+	fn create_surface(&self) -> SurfaceHandle<Self::Surface> {
+		let mut wrapper = || {
+			let document = window().expect("should have a window in this context").document().expect("window should have a document");
+
+			let canvas: HtmlCanvasElement = document.create_element("canvas")?.dyn_into::<HtmlCanvasElement>()?;
+			// TODO: replace "2d" with "bitmaprenderer" once we switch to ImageBitmap (lives on gpu) from ImageData (lives on cpu)
+			let context = canvas.get_context("2d").unwrap().unwrap().dyn_into::<CanvasRenderingContext2d>().unwrap();
+			let mut guard = self.ids.borrow_mut();
+			let id = SurfaceId(*guard);
+			*guard += 1;
+			self.canvases.borrow_mut().insert(id, context.clone());
+			// store the canvas in the global scope so it doesn't get garbage collected
+
+			let window = window().expect("should have a window in this context");
+			let window = Object::from(window);
+
+			let image_canvases_key = JsValue::from_str("imageCanvases");
+
+			let mut canvases = Reflect::get(&window, &image_canvases_key);
+			if let Err(e) = canvases {
+				Reflect::set(&JsValue::from(web_sys::window().unwrap()), &image_canvases_key, &Object::new()).unwrap();
+				canvases = Reflect::get(&window, &image_canvases_key);
+			}
+
+			// Convert key and value to JsValue
+			let js_key = JsValue::from_str(format!("canvas{}", id.0).as_str());
+			let js_value = JsValue::from(context.clone());
+
+			let canvases = Object::from(canvases.unwrap());
+
+			// Use Reflect API to set property
+			Reflect::set(&canvases, &js_key, &js_value)?;
+			Ok::<_, JsValue>(SurfaceHandle {
+				surface_id: id,
+				surface: context,
+				application_io: self,
+			})
+		};
+
+		wrapper().expect("should be able to set canvas in global scope")
+	}
+
+	fn destroy_surface(&self, surface_id: SurfaceId) {
+		self.canvases.borrow_mut().remove(&surface_id);
+
+		let window = window().expect("should have a window in this context");
+		let window = Object::from(window);
+
+		let image_canvases_key = JsValue::from_str("imageCanvases");
+
+		let wrapper = || {
+			if let Ok(canvases) = Reflect::get(&window, &image_canvases_key) {
+				// Convert key and value to JsValue
+				let js_key = JsValue::from_str(format!("canvas{}", surface_id.0).as_str());
+
+				// Use Reflect API to set property
+				Reflect::delete_property(&canvases.into(), &js_key)?;
+			}
+			Ok::<_, JsValue>(())
+		};
+
+		wrapper().expect("should be able to set canvas in global scope")
+	}
+}
+
+pub type WasmSurfaceHandle<'a> = SurfaceHandle<'a, CanvasRenderingContext2d>;
+pub type WasmSurfaceHandleFrame<'a> = SurfaceHandleFrame<'a, CanvasRenderingContext2d>;
+
+pub struct CreateSurfaceNode {}
+
+#[node_macro::node_fn(CreateSurfaceNode)]
+fn create_surface_node<'a: 'input>(editor: &'a WasmEditorApi<'a>) -> Arc<SurfaceHandle<'a, CanvasRenderingContext2d>> {
+	editor.application_io.create_surface().into()
+}
+
+pub struct DrawImageFrameNode<Surface> {
+	surface_handle: Surface,
+}
+
+#[node_macro::node_fn(DrawImageFrameNode)]
+async fn draw_image_frame_node<'a: 'input>(image: ImageFrame<SRGBA8>, surface_handle: Arc<SurfaceHandle<'a, CanvasRenderingContext2d>>) -> SurfaceHandleFrame<'a, CanvasRenderingContext2d> {
+	let image_data = image.image.data;
+	let array: Clamped<&[u8]> = Clamped(bytemuck::cast_slice(image_data.as_slice()));
+	if image.image.width > 0 && image.image.height > 0 {
+		let canvas = surface_handle.surface.canvas().expect("Failed to get canvas");
+		canvas.set_width(image.image.width);
+		canvas.set_height(image.image.height);
+		let image_data = web_sys::ImageData::new_with_u8_clamped_array_and_sh(array, image.image.width as u32, image.image.height as u32).expect("Failed to construct ImageData");
+		surface_handle.surface.put_image_data(&image_data, 0.0, 0.0).unwrap();
+	}
+	SurfaceHandleFrame {
+		surface_handle: surface_handle.into(),
+		transform: image.transform,
+	}
+}

node-graph/gcore/src/lib.rs

@@ -33,6 +33,8 @@ pub use graphic_element::*;
 #[cfg(feature = "alloc")]
 pub mod vector;
 
+pub mod application_io;
+
 pub mod quantization;
 
 use core::any::TypeId;
@@ -142,5 +144,6 @@ impl<'i, I: 'i, O: 'i> Node<'i, I> for Pin<&'i (dyn NodeIO<'i, I, Output = O> +
 	}
 }
 
-#[cfg(feature = "alloc")]
-pub use crate::raster::image::{EditorApi, ExtractImageFrame};
+pub use crate::application_io::{ExtractImageFrame, SurfaceFrame, SurfaceId};
+#[cfg(feature = "wasm")]
+pub use application_io::{wasm_application_io, wasm_application_io::WasmEditorApi as EditorApi};

node-graph/gcore/src/ops.rs

@@ -22,7 +22,7 @@ pub struct AddParameterNode<Second> {
 	second: Second,
 }
 
-#[node_macro::node_new(AddParameterNode)]
+#[node_macro::node_fn(AddParameterNode)]
 fn add_parameter<U, T>(first: U, second: T) -> <U as Add<T>>::Output
 where
 	U: Add<T>,
@@ -30,24 +30,6 @@ where
 	first + second
 }
 
-#[automatically_derived]
-impl<'input, U: 'input, T: 'input, S0: 'input> Node<'input, U> for AddParameterNode<S0>
-where
-	U: Add<T>,
-	S0: Node<'input, (), Output = T>,
-{
-	type Output = <U as Add<T>>::Output;
-	#[inline]
-	fn eval(&'input self, first: U) -> Self::Output {
-		let second = self.second.eval(());
-		{
-			{
-				first + second
-			}
-		}
-	}
-}
-
 pub struct MulParameterNode<Second> {
 	second: Second,
 }
@@ -224,6 +206,18 @@ where
 	}
 }
 
+pub struct IntoNode<I, O> {
+	_i: PhantomData<I>,
+	_o: PhantomData<O>,
+}
+#[node_macro::node_fn(IntoNode<_I, _O>)]
+fn into<_I, _O>(input: _I) -> _O
+where
+	_I: Into<_O>,
+{
+	input.into()
+}
+
 #[cfg(test)]
 mod test {
 	use super::*;

node-graph/gcore/src/raster.rs

@@ -5,7 +5,7 @@ use crate::Node;
 use bytemuck::{Pod, Zeroable};
 use glam::DVec2;
 
-pub use self::color::{Color, Luma};
+pub use self::color::{Color, Luma, SRGBA8};
 
 pub mod adjustments;
 pub mod bbox;

node-graph/gcore/src/raster/adjustments.rs

@@ -451,8 +451,8 @@ pub struct BlendNode<BlendMode, Opacity> {
 }
 
 #[node_macro::node_fn(BlendNode)]
-fn blend_node(input: (Color, Color), blend_mode: BlendMode, opacity: f32) -> Color {
-	let opacity = opacity / 100.;
+fn blend_node(input: (Color, Color), blend_mode: BlendMode, opacity: f64) -> Color {
+	let opacity = opacity as f32 / 100.;
 
 	let (foreground, background) = input;
 

node-graph/gcore/src/raster/color.rs

@@ -12,7 +12,92 @@ use spirv_std::num_traits::Euclid;
 
 use bytemuck::{Pod, Zeroable};
 
-use super::{Alpha, AssociatedAlpha, Luminance, Pixel, RGBMut, Rec709Primaries, RGB, SRGB};
+use super::{
+	discrete_srgb::{float_to_srgb_u8, srgb_u8_to_float},
+	Alpha, AssociatedAlpha, Luminance, Pixel, RGBMut, Rec709Primaries, RGB, SRGB,
+};
+
+#[repr(C)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+#[cfg_attr(feature = "std", derive(specta::Type))]
+#[derive(Debug, Default, Clone, Copy, PartialEq, DynAny, Pod, Zeroable)]
+pub struct SRGBA8 {
+	red: u8,
+	green: u8,
+	blue: u8,
+	alpha: u8,
+}
+
+impl From<Color> for SRGBA8 {
+	#[inline(always)]
+	fn from(c: Color) -> Self {
+		Self {
+			red: float_to_srgb_u8(c.r()),
+			green: float_to_srgb_u8(c.g()),
+			blue: float_to_srgb_u8(c.b()),
+			alpha: (c.a() * 255.0) as u8,
+		}
+	}
+}
+
+impl From<SRGBA8> for Color {
+	#[inline(always)]
+	fn from(color: SRGBA8) -> Self {
+		Self {
+			red: srgb_u8_to_float(color.red),
+			green: srgb_u8_to_float(color.green),
+			blue: srgb_u8_to_float(color.blue),
+			alpha: color.alpha as f32 / 255.0,
+		}
+	}
+}
+
+impl Luminance for SRGBA8 {
+	type LuminanceChannel = f32;
+	#[inline(always)]
+	fn luminance(&self) -> f32 {
+		// TODO: verify this is correct for sRGB
+		0.2126 * self.red() + 0.7152 * self.green() + 0.0722 * self.blue()
+	}
+}
+
+impl RGB for SRGBA8 {
+	type ColorChannel = f32;
+	#[inline(always)]
+	fn red(&self) -> f32 {
+		self.red as f32 / 255.0
+	}
+	#[inline(always)]
+	fn green(&self) -> f32 {
+		self.green as f32 / 255.0
+	}
+	#[inline(always)]
+	fn blue(&self) -> f32 {
+		self.blue as f32 / 255.0
+	}
+}
+
+impl Rec709Primaries for SRGBA8 {}
+impl SRGB for SRGBA8 {}
+
+impl Alpha for SRGBA8 {
+	type AlphaChannel = f32;
+	#[inline(always)]
+	fn alpha(&self) -> f32 {
+		self.alpha as f32 / 255.0
+	}
+
+	const TRANSPARENT: Self = SRGBA8 { red: 0, green: 0, blue: 0, alpha: 0 };
+
+	fn multiplied_alpha(&self, alpha: Self::AlphaChannel) -> Self {
+		let alpha = alpha * 255.0;
+		let mut result = *self;
+		result.alpha = (alpha * self.alpha()) as u8;
+		result
+	}
+}
+
+impl Pixel for SRGBA8 {}
 
 #[repr(C)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]

node-graph/gcore/src/raster/image.rs

@@ -325,44 +325,43 @@ impl<P: Hash + Pixel> Hash for ImageFrame<P> {
 	}
 }
 
-use crate::text::FontCache;
-#[derive(Clone, Debug, Hash, PartialEq)]
-#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
-pub struct EditorApi<'a> {
-	#[cfg_attr(feature = "serde", serde(skip))]
-	pub image_frame: Option<ImageFrame<Color>>,
-	#[cfg_attr(feature = "serde", serde(skip))]
-	pub font_cache: Option<&'a FontCache>,
-}
+/* This does not work because of missing specialization
+ * so we have to manually implement this for now
+impl<S: Into<P> + Pixel, P: Pixel> From<Image<S>> for Image<P> {
+	fn from(image: Image<S>) -> Self {
+		let data = image.data.into_iter().map(|x| x.into()).collect();
+		Self {
+			data,
+			width: image.width,
+			height: image.height,
+		}
+	}
+}*/
 
-unsafe impl StaticType for EditorApi<'_> {
-	type Static = EditorApi<'static>;
-}
-
-impl EditorApi<'_> {
-	pub fn empty() -> Self {
-		Self { image_frame: None, font_cache: None }
+impl From<ImageFrame<Color>> for ImageFrame<SRGBA8> {
+	fn from(image: ImageFrame<Color>) -> Self {
+		let data = image.image.data.into_iter().map(|x| x.into()).collect();
+		Self {
+			image: Image {
+				data,
+				width: image.image.width,
+				height: image.image.height,
+			},
+			transform: image.transform,
+		}
 	}
 }
 
-impl<'a> AsRef<EditorApi<'a>> for EditorApi<'a> {
-	fn as_ref(&self) -> &EditorApi<'a> {
-		self
-	}
-}
-
-#[derive(Debug, Clone, Copy, Default)]
-pub struct ExtractImageFrame;
-
-impl<'a: 'input, 'input> Node<'input, &'a EditorApi<'a>> for ExtractImageFrame {
-	type Output = ImageFrame<Color>;
-	fn eval(&'input self, editor_api: &'a EditorApi<'a>) -> Self::Output {
-		editor_api.image_frame.clone().unwrap_or(ImageFrame::identity())
-	}
-}
-
-impl ExtractImageFrame {
-	pub fn new() -> Self {
-		Self
+impl From<ImageFrame<SRGBA8>> for ImageFrame<Color> {
+	fn from(image: ImageFrame<SRGBA8>) -> Self {
+		let data = image.image.data.into_iter().map(|x| x.into()).collect();
+		Self {
+			image: Image {
+				data,
+				width: image.image.width,
+				height: image.image.height,
+			},
+			transform: image.transform,
+		}
 	}
 }

node-graph/gcore/src/text.rs

@@ -16,6 +16,6 @@ pub struct TextGenerator<Text, FontName, Size> {
 
 #[node_fn(TextGenerator)]
 fn generate_text<'a: 'input>(editor: &'a EditorApi<'a>, text: String, font_name: Font, font_size: f64) -> crate::vector::VectorData {
-	let buzz_face = editor.font_cache.and_then(|cache| cache.get(&font_name)).map(|data| load_face(data));
+	let buzz_face = editor.font_cache.get(&font_name).map(|data| load_face(data));
 	crate::vector::VectorData::from_subpaths(to_path(&text, buzz_face, font_size, None))
 }