Integrate raw WGPU textures into Vello rendering pipeline (#1897)

* WIP

* Introduce `Raster` enum and plumb texture injection to vello

* Use correct texture formas for usage in vello

* Add missing cache implementations

* Update vello image override api

* Use git version of vello

* Fix type for upload texture node

* Improve crash resiliance

* Fix warnings

* Remove unused node impls

node-graph/gcore/Cargo.toml

@@ -14,7 +14,8 @@ nightly = []
 alloc = ["dyn-any", "bezier-rs"]
 type_id_logging = []
 wasm = ["web-sys"]
-vello = ["dep:vello", "bezier-rs/kurbo"]
+wgpu = ["dep:wgpu"]
+vello = ["dep:vello", "bezier-rs/kurbo", "wgpu"]
 std = [
 	"dyn-any",
 	"dyn-any/std",
@@ -59,6 +60,7 @@ bezier-rs = { workspace = true, optional = true }
 kurbo = { workspace = true, optional = true }
 base64 = { workspace = true, optional = true }
 vello = { workspace  = true, optional = true }
+wgpu = { workspace = true, optional = true }
 specta = { workspace = true, optional = true }
 rustybuzz = { workspace = true, optional = true }
 wasm-bindgen = { workspace = true, optional = true }

node-graph/gcore/src/application_io.rs

@@ -63,6 +63,55 @@ impl Size for web_sys::HtmlCanvasElement {
 	}
 }
 
+#[derive(Debug, Clone)]
+pub struct TextureFrame {
+	#[cfg(feature = "wgpu")]
+	pub texture: Arc<wgpu::Texture>,
+	#[cfg(not(feature = "wgpu"))]
+	pub texture: (),
+	pub transform: DAffine2,
+}
+
+impl Hash for TextureFrame {
+	fn hash<H: Hasher>(&self, state: &mut H) {
+		self.transform.to_cols_array().iter().for_each(|x| x.to_bits().hash(state));
+		#[cfg(feature = "wgpu")]
+		self.texture.global_id().hash(state);
+	}
+}
+
+impl PartialEq for TextureFrame {
+	fn eq(&self, other: &Self) -> bool {
+		#[cfg(feature = "wgpu")]
+		return self.transform.eq(&other.transform) && self.texture.global_id() == other.texture.global_id();
+
+		#[cfg(not(feature = "wgpu"))]
+		self.transform.eq(&other.transform)
+	}
+}
+
+impl Transform for TextureFrame {
+	fn transform(&self) -> DAffine2 {
+		self.transform
+	}
+}
+impl TransformMut for TextureFrame {
+	fn transform_mut(&mut self) -> &mut DAffine2 {
+		&mut self.transform
+	}
+}
+
+unsafe impl StaticType for TextureFrame {
+	type Static = TextureFrame;
+}
+
+#[cfg(feature = "wgpu")]
+impl Size for TextureFrame {
+	fn size(&self) -> UVec2 {
+		UVec2::new(self.texture.width(), self.texture.height())
+	}
+}
+
 impl<S: Size> From<SurfaceHandleFrame<S>> for SurfaceFrame {
 	fn from(x: SurfaceHandleFrame<S>) -> Self {
 		Self {

node-graph/gcore/src/graphic_element.rs

@@ -1,15 +1,14 @@
-use crate::application_io::SurfaceHandleFrame;
+use crate::application_io::TextureFrame;
 use crate::raster::{BlendMode, ImageFrame};
-use crate::transform::Footprint;
+use crate::transform::{Footprint, Transform, TransformMut};
 use crate::vector::VectorData;
-use crate::{Color, Node, SurfaceFrame};
+use crate::{Color, Node};
 
 use dyn_any::{DynAny, StaticType};
 use node_macro::node_fn;
 
 use core::ops::{Deref, DerefMut};
-use glam::{DAffine2, IVec2, UVec2};
-use web_sys::HtmlCanvasElement;
+use glam::{DAffine2, IVec2};
 
 pub mod renderer;
 
@@ -65,10 +64,7 @@ pub enum GraphicElement {
 	GraphicGroup(GraphicGroup),
 	/// A vector shape, equivalent to the SVG <path> tag: https://developer.mozilla.org/en-US/docs/Web/SVG/Element/path
 	VectorData(Box<VectorData>),
-	/// A bitmap image with a finite position and extent, equivalent to the SVG <image> tag: https://developer.mozilla.org/en-US/docs/Web/SVG/Element/image
-	ImageFrame(ImageFrame<Color>),
-	/// A Canvas element
-	Surface(SurfaceFrame),
+	Raster(Raster),
 }
 
 // TODO: Can this be removed? It doesn't necessarily make that much sense to have a default when, instead, the entire GraphicElement just shouldn't exist if there's no specific content to assign it.
@@ -78,6 +74,58 @@ impl Default for GraphicElement {
 	}
 }
 
+#[derive(Clone, Debug, Hash, PartialEq, DynAny)]
+pub enum Raster {
+	/// A bitmap image with a finite position and extent, equivalent to the SVG <image> tag: https://developer.mozilla.org/en-US/docs/Web/SVG/Element/image
+	ImageFrame(ImageFrame<Color>),
+	Texture(TextureFrame),
+}
+
+impl<'de> serde::Deserialize<'de> for Raster {
+	fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+	where
+		D: serde::Deserializer<'de>,
+	{
+		let frame = ImageFrame::deserialize(deserializer)?;
+		Ok(Raster::ImageFrame(frame))
+	}
+}
+
+impl serde::Serialize for Raster {
+	fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+	where
+		S: serde::Serializer,
+	{
+		match self {
+			Raster::ImageFrame(_) => self.serialize(serializer),
+			Raster::Texture(_) => todo!(),
+		}
+	}
+}
+
+impl Transform for Raster {
+	fn transform(&self) -> DAffine2 {
+		match self {
+			Raster::ImageFrame(frame) => frame.transform(),
+			Raster::Texture(frame) => frame.transform(),
+		}
+	}
+	fn local_pivot(&self, pivot: glam::DVec2) -> glam::DVec2 {
+		match self {
+			Raster::ImageFrame(frame) => frame.local_pivot(pivot),
+			Raster::Texture(frame) => frame.local_pivot(pivot),
+		}
+	}
+}
+impl TransformMut for Raster {
+	fn transform_mut(&mut self) -> &mut DAffine2 {
+		match self {
+			Raster::ImageFrame(frame) => frame.transform_mut(),
+			Raster::Texture(frame) => frame.transform_mut(),
+		}
+	}
+}
+
 /// Some [`ArtboardData`] with some optional clipping bounds that can be exported.
 /// Similar to an Inkscape page: https://media.inkscape.org/media/doc/release_notes/1.2/Inkscape_1.2.html#Page_tool
 #[derive(Clone, Debug, Hash, PartialEq, DynAny)]
@@ -202,7 +250,12 @@ async fn add_artboard<Data: Into<Artboard> + Send>(footprint: Footprint, artboar
 
 impl From<ImageFrame<Color>> for GraphicElement {
 	fn from(image_frame: ImageFrame<Color>) -> Self {
-		GraphicElement::ImageFrame(image_frame)
+		GraphicElement::Raster(Raster::ImageFrame(image_frame))
+	}
+}
+impl From<TextureFrame> for GraphicElement {
+	fn from(texture: TextureFrame) -> Self {
+		GraphicElement::Raster(Raster::Texture(texture))
 	}
 }
 impl From<VectorData> for GraphicElement {
@@ -215,39 +268,6 @@ impl From<GraphicGroup> for GraphicElement {
 		GraphicElement::GraphicGroup(graphic_group)
 	}
 }
-impl From<SurfaceFrame> for GraphicElement {
-	fn from(surface: SurfaceFrame) -> Self {
-		GraphicElement::Surface(surface)
-	}
-}
-impl From<alloc::sync::Arc<SurfaceHandleFrame<HtmlCanvasElement>>> for GraphicElement {
-	fn from(surface: alloc::sync::Arc<SurfaceHandleFrame<HtmlCanvasElement>>) -> Self {
-		let surface_id = surface.surface_handle.window_id;
-		let transform = surface.transform;
-		GraphicElement::Surface(SurfaceFrame {
-			surface_id,
-			transform,
-			resolution: UVec2 {
-				x: surface.surface_handle.surface.width(),
-				y: surface.surface_handle.surface.height(),
-			},
-		})
-	}
-}
-impl From<SurfaceHandleFrame<HtmlCanvasElement>> for GraphicElement {
-	fn from(surface: SurfaceHandleFrame<HtmlCanvasElement>) -> Self {
-		let surface_id = surface.surface_handle.window_id;
-		let transform = surface.transform;
-		GraphicElement::Surface(SurfaceFrame {
-			surface_id,
-			transform,
-			resolution: UVec2 {
-				x: surface.surface_handle.surface.width(),
-				y: surface.surface_handle.surface.height(),
-			},
-		})
-	}
-}
 
 impl Deref for GraphicGroup {
 	type Target = Vec<GraphicElement>;

node-graph/gcore/src/graphic_element/renderer.rs

@@ -3,13 +3,12 @@ mod rect;
 pub use quad::Quad;
 pub use rect::Rect;
 
-use crate::raster::bbox::Bbox;
 use crate::raster::{BlendMode, Image, ImageFrame};
 use crate::transform::Transform;
 use crate::uuid::generate_uuid;
 use crate::vector::style::{Fill, Stroke, ViewMode};
 use crate::vector::PointId;
-use crate::SurfaceFrame;
+use crate::Raster;
 use crate::{vector::VectorData, Artboard, Color, GraphicElement, GraphicGroup};
 
 use bezier_rs::Subpath;
@@ -219,6 +218,12 @@ pub enum ImageRenderMode {
 	Base64,
 }
 
+#[derive(Clone, Debug, Default)]
+pub struct RenderContext {
+	#[cfg(feature = "wgpu")]
+	pub ressource_overrides: std::collections::HashMap<u64, alloc::sync::Arc<wgpu::Texture>>,
+}
+
 /// Static state used whilst rendering
 #[derive(Default)]
 pub struct RenderParams {
@@ -268,13 +273,13 @@ pub trait GraphicElementRendered {
 	fn bounding_box(&self, transform: DAffine2) -> Option<[DVec2; 2]>;
 	fn add_click_targets(&self, click_targets: &mut Vec<ClickTarget>);
 	#[cfg(feature = "vello")]
-	fn to_vello_scene(&self, transform: DAffine2) -> Scene {
+	fn to_vello_scene(&self, transform: DAffine2, context: &mut RenderContext) -> Scene {
 		let mut scene = vello::Scene::new();
-		self.render_to_vello(&mut scene, transform);
+		self.render_to_vello(&mut scene, transform, context);
 		scene
 	}
 	#[cfg(feature = "vello")]
-	fn render_to_vello(&self, _scene: &mut Scene, _transform: DAffine2) {}
+	fn render_to_vello(&self, _scene: &mut Scene, _transform: DAffine2, _render_condext: &mut RenderContext) {}
 
 	fn contains_artboard(&self) -> bool {
 		false
@@ -323,7 +328,7 @@ impl GraphicElementRendered for GraphicGroup {
 	}
 
 	#[cfg(feature = "vello")]
-	fn render_to_vello(&self, scene: &mut Scene, transform: DAffine2) {
+	fn render_to_vello(&self, scene: &mut Scene, transform: DAffine2, context: &mut RenderContext) {
 		let child_transform = transform * self.transform;
 
 		let Some(bounds) = self.bounding_box(transform) else { return };
@@ -340,7 +345,7 @@ impl GraphicElementRendered for GraphicGroup {
 			);
 		}
 		for element in self.iter() {
-			element.render_to_vello(scene, child_transform);
+			element.render_to_vello(scene, child_transform, context);
 		}
 		if layer {
 			scene.pop_layer();
@@ -406,7 +411,7 @@ impl GraphicElementRendered for VectorData {
 	}
 
 	#[cfg(feature = "vello")]
-	fn render_to_vello(&self, scene: &mut Scene, transform: DAffine2) {
+	fn render_to_vello(&self, scene: &mut Scene, transform: DAffine2, _: &mut RenderContext) {
 		use crate::vector::style::GradientType;
 		use vello::peniko;
 
@@ -593,7 +598,7 @@ impl GraphicElementRendered for Artboard {
 	}
 
 	#[cfg(feature = "vello")]
-	fn render_to_vello(&self, scene: &mut Scene, transform: DAffine2) {
+	fn render_to_vello(&self, scene: &mut Scene, transform: DAffine2, context: &mut RenderContext) {
 		use vello::peniko;
 
 		// Render background
@@ -608,7 +613,7 @@ impl GraphicElementRendered for Artboard {
 		if self.clip {
 			scene.push_layer(blend_mode, 1., kurbo::Affine::new(transform.to_cols_array()), &rect);
 		}
-		self.graphic_group.render_to_vello(scene, transform);
+		self.graphic_group.render_to_vello(scene, transform, context);
 		if self.clip {
 			scene.pop_layer();
 		}
@@ -643,9 +648,9 @@ impl GraphicElementRendered for crate::ArtboardGroup {
 	}
 
 	#[cfg(feature = "vello")]
-	fn render_to_vello(&self, scene: &mut Scene, transform: DAffine2) {
+	fn render_to_vello(&self, scene: &mut Scene, transform: DAffine2, context: &mut RenderContext) {
 		for artboard in &self.artboards {
-			artboard.render_to_vello(scene, transform)
+			artboard.render_to_vello(scene, transform, context)
 		}
 	}
 
@@ -654,42 +659,6 @@ impl GraphicElementRendered for crate::ArtboardGroup {
 	}
 }
 
-impl GraphicElementRendered for SurfaceFrame {
-	fn render_svg(&self, render: &mut SvgRender, _render_params: &RenderParams) {
-		let transform = self.transform;
-
-		let (width, height) = (transform.transform_vector2(DVec2::new(1., 0.)).length(), transform.transform_vector2(DVec2::new(0., 1.)).length());
-
-		let matrix = format_transform_matrix(transform * DAffine2::from_scale((width, height).into()).inverse());
-		let transform = if matrix.is_empty() { String::new() } else { format!(r#" transform="{}""#, matrix) };
-
-		let canvas = format!(
-			r#"<foreignObject width="{}" height="{}"{transform}><div data-canvas-placeholder="canvas{}"></div></foreignObject>"#,
-			width.abs(),
-			height.abs(),
-			self.surface_id
-		);
-		render.svg.push(canvas.into())
-	}
-
-	#[cfg(feature = "vello")]
-	fn render_to_vello(&self, _scene: &mut Scene, _transform: DAffine2) {
-		todo!()
-	}
-
-	fn bounding_box(&self, transform: DAffine2) -> Option<[DVec2; 2]> {
-		let bbox = Bbox::from_transform(transform);
-		let aabb = bbox.to_axis_aligned_bbox();
-		Some([aabb.start, aabb.end])
-	}
-
-	fn add_click_targets(&self, _click_targets: &mut Vec<ClickTarget>) {}
-
-	fn contains_artboard(&self) -> bool {
-		false
-	}
-}
-
 impl GraphicElementRendered for ImageFrame<Color> {
 	fn render_svg(&self, render: &mut SvgRender, render_params: &RenderParams) {
 		let transform = self.transform * render.transform;
@@ -737,7 +706,7 @@ impl GraphicElementRendered for ImageFrame<Color> {
 	}
 
 	#[cfg(feature = "vello")]
-	fn render_to_vello(&self, scene: &mut Scene, transform: DAffine2) {
+	fn render_to_vello(&self, scene: &mut Scene, transform: DAffine2, _: &mut RenderContext) {
 		use vello::peniko;
 
 		let image = &self.image;
@@ -756,51 +725,135 @@ impl GraphicElementRendered for ImageFrame<Color> {
 		scene.draw_image(&image, vello::kurbo::Affine::new(transform.to_cols_array()));
 	}
 }
+impl GraphicElementRendered for Raster {
+	fn render_svg(&self, render: &mut SvgRender, render_params: &RenderParams) {
+		let transform = self.transform() * render.transform;
+
+		match render_params.image_render_mode {
+			ImageRenderMode::Base64 => {
+				let image = match self {
+					Raster::ImageFrame(ref image) => image,
+					Raster::Texture(_) => return,
+				};
+				let (image, blending) = (&image.image, image.alpha_blending);
+				if image.data.is_empty() {
+					return;
+				}
+
+				let base64_string = image.base64_string.clone().unwrap_or_else(|| {
+					let output = image.to_png();
+					let preamble = "data:image/png;base64,";
+					let mut base64_string = String::with_capacity(preamble.len() + output.len() * 4);
+					base64_string.push_str(preamble);
+					base64::engine::general_purpose::STANDARD.encode_string(output, &mut base64_string);
+					base64_string
+				});
+				render.leaf_tag("image", |attributes| {
+					attributes.push("width", 1.to_string());
+					attributes.push("height", 1.to_string());
+					attributes.push("preserveAspectRatio", "none");
+					attributes.push("href", base64_string);
+					let matrix = format_transform_matrix(transform);
+					if !matrix.is_empty() {
+						attributes.push("transform", matrix);
+					}
+					if blending.blend_mode != BlendMode::default() {
+						attributes.push("style", blending.blend_mode.render());
+					}
+				});
+			}
+		}
+	}
+
+	fn bounding_box(&self, transform: DAffine2) -> Option<[DVec2; 2]> {
+		let transform = self.transform() * transform;
+		(transform.matrix2 != glam::DMat2::ZERO).then(|| (transform * Quad::from_box([DVec2::ZERO, DVec2::ONE])).bounding_box())
+	}
+
+	fn add_click_targets(&self, click_targets: &mut Vec<ClickTarget>) {
+		let subpath = Subpath::new_rect(DVec2::ZERO, DVec2::ONE);
+		click_targets.push(ClickTarget::new(subpath, 0.));
+	}
+
+	#[cfg(feature = "vello")]
+	fn render_to_vello(&self, scene: &mut Scene, transform: DAffine2, context: &mut RenderContext) {
+		use vello::peniko;
+
+		match self {
+			Raster::ImageFrame(image_frame) => {
+				let image = &image_frame.image;
+				if image.data.is_empty() {
+					return;
+				}
+				let image = vello::peniko::Image {
+					data: image.to_flat_u8().0.into(),
+					width: image.width,
+					height: image.height,
+					format: peniko::Format::Rgba8,
+					extend: peniko::Extend::Repeat,
+				};
+				let transform = transform * self.transform() * DAffine2::from_scale(1. / DVec2::new(image.width as f64, image.height as f64));
+
+				scene.draw_image(&image, vello::kurbo::Affine::new(transform.to_cols_array()));
+			}
+			Raster::Texture(texture) => {
+				let image = vello::peniko::Image {
+					data: vec![].into(),
+					width: texture.texture.width(),
+					height: texture.texture.height(),
+					format: peniko::Format::Rgba8,
+					extend: peniko::Extend::Repeat,
+				};
+				let id = image.data.id();
+				context.ressource_overrides.insert(id, texture.texture.clone());
+
+				let transform = transform * self.transform() * DAffine2::from_scale(1. / DVec2::new(image.width as f64, image.height as f64));
+
+				scene.draw_image(&image, vello::kurbo::Affine::new(transform.to_cols_array()));
+			}
+		};
+	}
+}
 
 impl GraphicElementRendered for GraphicElement {
 	fn render_svg(&self, render: &mut SvgRender, render_params: &RenderParams) {
 		match self {
 			GraphicElement::VectorData(vector_data) => vector_data.render_svg(render, render_params),
-			GraphicElement::ImageFrame(image_frame) => image_frame.render_svg(render, render_params),
+			GraphicElement::Raster(raster) => raster.render_svg(render, render_params),
 			GraphicElement::GraphicGroup(graphic_group) => graphic_group.render_svg(render, render_params),
-			GraphicElement::Surface(surface) => surface.render_svg(render, render_params),
 		}
 	}
 
 	fn bounding_box(&self, transform: DAffine2) -> Option<[DVec2; 2]> {
 		match self {
 			GraphicElement::VectorData(vector_data) => GraphicElementRendered::bounding_box(&**vector_data, transform),
-			GraphicElement::ImageFrame(image_frame) => image_frame.bounding_box(transform),
+			GraphicElement::Raster(raster) => raster.bounding_box(transform),
 			GraphicElement::GraphicGroup(graphic_group) => graphic_group.bounding_box(transform),
-			GraphicElement::Surface(surface) => surface.bounding_box(transform),
 		}
 	}
 
 	fn add_click_targets(&self, click_targets: &mut Vec<ClickTarget>) {
 		match self {
 			GraphicElement::VectorData(vector_data) => vector_data.add_click_targets(click_targets),
-			GraphicElement::ImageFrame(image_frame) => image_frame.add_click_targets(click_targets),
+			GraphicElement::Raster(raster) => raster.add_click_targets(click_targets),
 			GraphicElement::GraphicGroup(graphic_group) => graphic_group.add_click_targets(click_targets),
-			GraphicElement::Surface(surface) => surface.add_click_targets(click_targets),
 		}
 	}
 
 	#[cfg(feature = "vello")]
-	fn render_to_vello(&self, scene: &mut Scene, transform: DAffine2) {
+	fn render_to_vello(&self, scene: &mut Scene, transform: DAffine2, context: &mut RenderContext) {
 		match self {
-			GraphicElement::VectorData(vector_data) => vector_data.render_to_vello(scene, transform),
-			GraphicElement::ImageFrame(image_frame) => image_frame.render_to_vello(scene, transform),
-			GraphicElement::GraphicGroup(graphic_group) => graphic_group.render_to_vello(scene, transform),
-			GraphicElement::Surface(surface) => surface.render_to_vello(scene, transform),
+			GraphicElement::VectorData(vector_data) => vector_data.render_to_vello(scene, transform, context),
+			GraphicElement::GraphicGroup(graphic_group) => graphic_group.render_to_vello(scene, transform, context),
+			GraphicElement::Raster(raster) => raster.render_to_vello(scene, transform, context),
 		}
 	}
 
 	fn contains_artboard(&self) -> bool {
 		match self {
 			GraphicElement::VectorData(vector_data) => vector_data.contains_artboard(),
-			GraphicElement::ImageFrame(image_frame) => image_frame.contains_artboard(),
 			GraphicElement::GraphicGroup(graphic_group) => graphic_group.contains_artboard(),
-			GraphicElement::Surface(surface) => surface.contains_artboard(),
+			GraphicElement::Raster(raster) => raster.contains_artboard(),
 		}
 	}
 }

node-graph/gcore/src/transform.rs

@@ -65,17 +65,15 @@ impl Transform for GraphicElement {
 	fn transform(&self) -> DAffine2 {
 		match self {
 			GraphicElement::VectorData(vector_shape) => vector_shape.transform(),
-			GraphicElement::ImageFrame(image_frame) => image_frame.transform(),
 			GraphicElement::GraphicGroup(graphic_group) => graphic_group.transform(),
-			GraphicElement::Surface(surface) => surface.transform(),
+			GraphicElement::Raster(raster) => raster.transform(),
 		}
 	}
 	fn local_pivot(&self, pivot: DVec2) -> DVec2 {
 		match self {
 			GraphicElement::VectorData(vector_shape) => vector_shape.local_pivot(pivot),
-			GraphicElement::ImageFrame(image_frame) => image_frame.local_pivot(pivot),
 			GraphicElement::GraphicGroup(graphic_group) => graphic_group.local_pivot(pivot),
-			GraphicElement::Surface(surface) => surface.local_pivot(pivot),
+			GraphicElement::Raster(raster) => raster.local_pivot(pivot),
 		}
 	}
 }
@@ -83,9 +81,8 @@ impl TransformMut for GraphicElement {
 	fn transform_mut(&mut self) -> &mut DAffine2 {
 		match self {
 			GraphicElement::VectorData(vector_shape) => vector_shape.transform_mut(),
-			GraphicElement::ImageFrame(image_frame) => image_frame.transform_mut(),
 			GraphicElement::GraphicGroup(graphic_group) => graphic_group.transform_mut(),
-			GraphicElement::Surface(surface) => surface.transform_mut(),
+			GraphicElement::Raster(raster) => raster.transform_mut(),
 		}
 	}
 }

node-graph/gstd/src/raster.rs

@@ -199,7 +199,7 @@ pub struct MaskImageNode<P, S, Stencil> {
 fn mask_image<
 	// _P is the color of the input image. It must have an alpha channel because that is going to
 	// be modified by the mask
-	_P: Copy + Alpha,
+	_P: Alpha,
 	// _S is the color of the stencil. It must have a luminance channel because that is used to
 	// mask the input image
 	_S: Luminance,

node-graph/gstd/src/vector.rs

@@ -74,8 +74,7 @@ fn boolean_operation_node(graphic_group: GraphicGroup, boolean_operation: Boolea
 				let vector_data = collect_vector_data(graphic_group);
 				boolean_operation_on_vector_data(&vector_data, BooleanOperation::Union)
 			}
-			GraphicElement::ImageFrame(image) => vector_from_image(image),
-			GraphicElement::Surface(image) => vector_from_image(image),
+			GraphicElement::Raster(image) => vector_from_image(image),
 		}
 	}
 

node-graph/gstd/src/wasm_application_io.rs

@@ -120,12 +120,13 @@ async fn render_canvas(render_config: RenderConfig, data: impl GraphicElementRen
 		let mut scene = Scene::new();
 		let mut child = Scene::new();
 
-		data.render_to_vello(&mut child, glam::DAffine2::IDENTITY);
+		let mut context = wgpu_executor::RenderContext::default();
+		data.render_to_vello(&mut child, glam::DAffine2::IDENTITY, &mut context);
 
 		// TODO: Instead of applying the transform here, pass the transform during the translation to avoid the O(Nr cost
 		scene.append(&child, Some(kurbo::Affine::new(footprint.transform.to_cols_array())));
 
-		exec.render_vello_scene(&scene, &surface_handle, footprint.resolution.x, footprint.resolution.y)
+		exec.render_vello_scene(&scene, &surface_handle, footprint.resolution.x, footprint.resolution.y, &context)
 			.await
 			.expect("Failed to render Vello scene");
 	} else {

node-graph/interpreted-executor/src/node_registry.rs

@@ -17,7 +17,7 @@ use graphene_core::{concrete, generic, Artboard, ArtboardGroup, GraphicGroup};
 use graphene_core::{Cow, ProtoNodeIdentifier, Type};
 use graphene_core::{Node, NodeIO, NodeIOTypes};
 use graphene_std::any::{ComposeTypeErased, DowncastBothNode, DynAnyNode, FutureWrapperNode, IntoTypeErasedNode};
-use graphene_std::application_io::RenderConfig;
+use graphene_std::application_io::{RenderConfig, TextureFrame};
 use graphene_std::raster::*;
 use graphene_std::wasm_application_io::*;
 use graphene_std::GraphicElement;
@@ -389,7 +389,7 @@ fn node_registry() -> HashMap<ProtoNodeIdentifier, HashMap<NodeIOTypes, NodeCons
 		async_node!(
 			wgpu_executor::UploadTextureNode<_>,
 			input: ImageFrame<Color>,
-			output: ShaderInputFrame,
+			output: TextureFrame,
 			params: [&WgpuExecutor]
 		),
 		#[cfg(feature = "gpu")]
@@ -608,6 +608,7 @@ fn node_registry() -> HashMap<ProtoNodeIdentifier, HashMap<NodeIOTypes, NodeCons
 		async_node!(graphene_core::memo::MemoNode<_, _>, input: (), output: QuantizationChannels, params: [QuantizationChannels]),
 		async_node!(graphene_core::memo::MemoNode<_, _>, input: (), output: Vec<DVec2>, params: [Vec<DVec2>]),
 		async_node!(graphene_core::memo::MemoNode<_, _>, input: (), output: Arc<WasmSurfaceHandle>, params: [Arc<WasmSurfaceHandle>]),
+		async_node!(graphene_core::memo::MemoNode<_, _>, input: (), output: Arc<TextureFrame>, params: [Arc<WasmSurfaceHandle>]),
 		async_node!(graphene_core::memo::MemoNode<_, _>, input: (), output: WindowHandle, params: [WindowHandle]),
 		#[cfg(feature = "gpu")]
 		async_node!(graphene_core::memo::MemoNode<_, _>, input: (), output: ShaderInputFrame, params: [ShaderInputFrame]),
@@ -639,6 +640,7 @@ fn node_registry() -> HashMap<ProtoNodeIdentifier, HashMap<NodeIOTypes, NodeCons
 		async_node!(graphene_core::memo::ImpureMemoNode<_, _, _>, input: Footprint, output: ShaderInputFrame, fn_params: [Footprint => ShaderInputFrame]),
 		async_node!(graphene_core::memo::ImpureMemoNode<_, _, _>, input: Footprint, output: WgpuSurface, fn_params: [Footprint => WgpuSurface]),
 		async_node!(graphene_core::memo::ImpureMemoNode<_, _, _>, input: Footprint, output: Option<WgpuSurface>, fn_params: [Footprint => Option<WgpuSurface>]),
+		async_node!(graphene_core::memo::ImpureMemoNode<_, _, _>, input: Footprint, output: TextureFrame, fn_params: [Footprint => TextureFrame]),
 		register_node!(graphene_core::structural::ConsNode<_, _>, input: Image<Color>, params: [&str]),
 		register_node!(graphene_std::raster::ImageFrameNode<_, _>, input: Image<Color>, params: [DAffine2]),
 		register_node!(graphene_std::raster::NoisePatternNode<_, _, _, _, _, _, _, _, _, _, _, _, _, _, _>, input: (), params: [UVec2, u32, f64, NoiseType, DomainWarpType, f64, FractalType, u32, f64, f64, f64, f64, CellularDistanceFunction, CellularReturnType, f64]),
@@ -665,6 +667,7 @@ fn node_registry() -> HashMap<ProtoNodeIdentifier, HashMap<NodeIOTypes, NodeCons
 		async_node!(graphene_core::transform::TransformNode<_, _, _, _, _, _>, input: Footprint, output: VectorData, fn_params: [Footprint => VectorData, () => DVec2, () => f64, () => DVec2, () => DVec2, () => DVec2]),
 		// async_node!(graphene_core::transform::TransformNode<_, _, _, _, _, _>, input: Footprint, output: WasmSurfaceHandleFrame, fn_params: [Footprint => WasmSurfaceHandleFrame, () => DVec2, () => f64, () => DVec2, () => DVec2, () => DVec2]),
 		async_node!(graphene_core::transform::TransformNode<_, _, _, _, _, _>, input: Footprint, output: ImageFrame<Color>, fn_params: [Footprint => ImageFrame<Color>, () => DVec2, () => f64, () => DVec2, () => DVec2, () => DVec2]),
+		async_node!(graphene_core::transform::TransformNode<_, _, _, _, _, _>, input: Footprint, output: TextureFrame, fn_params: [Footprint => TextureFrame, () => DVec2, () => f64, () => DVec2, () => DVec2, () => DVec2]),
 		async_node!(graphene_core::transform::TransformNode<_, _, _, _, _, _>, input: Footprint, output: GraphicGroup, fn_params: [Footprint => GraphicGroup, () => DVec2, () => f64, () => DVec2, () => DVec2, () => DVec2]),
 		register_node!(graphene_core::transform::SetTransformNode<_>, input: VectorData, params: [VectorData]),
 		register_node!(graphene_core::transform::SetTransformNode<_>, input: ImageFrame<Color>, params: [ImageFrame<Color>]),
@@ -771,11 +774,7 @@ fn node_registry() -> HashMap<ProtoNodeIdentifier, HashMap<NodeIOTypes, NodeCons
 		register_node!(graphene_core::ToGraphicElementNode, input: graphene_core::vector::VectorData, params: []),
 		register_node!(graphene_core::ToGraphicElementNode, input: ImageFrame<Color>, params: []),
 		register_node!(graphene_core::ToGraphicElementNode, input: GraphicGroup, params: []),
-		register_node!(graphene_core::ToGraphicElementNode, input: graphene_std::SurfaceFrame, params: []),
-		#[cfg(target_arch = "wasm32")]
-		register_node!(graphene_core::ToGraphicElementNode, input: Arc<graphene_std::application_io::SurfaceHandleFrame<wgpu::web_sys::HtmlCanvasElement>>, params: []),
-		#[cfg(target_arch = "wasm32")]
-		register_node!(graphene_core::ToGraphicElementNode, input: graphene_std::application_io::SurfaceHandleFrame<wgpu::web_sys::HtmlCanvasElement>, params: []),
+		register_node!(graphene_core::ToGraphicElementNode, input: TextureFrame, params: []),
 		register_node!(graphene_core::ToGraphicGroupNode, input: graphene_core::vector::VectorData, params: []),
 		register_node!(graphene_core::ToGraphicGroupNode, input: ImageFrame<Color>, params: []),
 		register_node!(graphene_core::ToGraphicGroupNode, input: GraphicGroup, params: []),

node-graph/wgpu-executor/Cargo.toml

@@ -14,7 +14,7 @@ passthrough = []
 gpu-executor = { path = "../gpu-executor" }
 
 # Workspace dependencies
-graphene-core = { workspace = true, features = ["std", "alloc", "gpu"] }
+graphene-core = { workspace = true, features = ["std", "alloc", "gpu", "wgpu"] }
 dyn-any = { workspace = true, features = ["log-bad-types", "rc", "glam"] }
 node-macro = { workspace = true }
 num-traits = { workspace = true }
@@ -26,7 +26,6 @@ anyhow = { workspace = true }
 wgpu = { workspace = true, features = [
 	"spirv",
 	"strict_asserts",
-	"api_log_info",
 ] }
 spirv = { workspace = true }
 futures = { workspace = true }

node-graph/wgpu-executor/src/context.rs

@@ -38,6 +38,7 @@ impl Context {
 					#[cfg(feature = "passthrough")]
 					required_features: wgpu::Features::SPIRV_SHADER_PASSTHROUGH,
 					required_limits,
+					memory_hints: Default::default(),
 				},
 				None,
 			)

node-graph/wgpu-executor/src/executor.rs

@@ -104,6 +104,7 @@ async fn execute_shader<I: Pod + Send + Sync, O: Pod + Send + Sync>(device: Arc<
 		module: &cs_module,
 		entry_point: entry_point.as_str(),
 		compilation_options: Default::default(),
+		cache: None,
 	});
 
 	// Instantiates the bind group, once again specifying the binding of buffers.

node-graph/wgpu-executor/src/lib.rs

@@ -6,9 +6,8 @@ pub use executor::GpuExecutor;
 
 use dyn_any::{DynAny, StaticType};
 use gpu_executor::{ComputePassDimensions, GPUConstant, StorageBufferOptions, TextureBufferOptions, TextureBufferType, ToStorageBuffer, ToUniformBuffer};
-use graphene_core::application_io::{ApplicationIo, EditorApi, SurfaceHandle};
-use graphene_core::raster::color::RGBA16F;
-use graphene_core::raster::{Image, ImageFrame};
+use graphene_core::application_io::{ApplicationIo, EditorApi, SurfaceHandle, TextureFrame};
+use graphene_core::raster::{Image, ImageFrame, SRGBA8};
 use graphene_core::transform::{Footprint, Transform};
 use graphene_core::Type;
 use graphene_core::{Color, Cow, Node, SurfaceFrame};
@@ -18,9 +17,9 @@ use futures::Future;
 use glam::{DAffine2, UVec2};
 use std::pin::Pin;
 use std::sync::Arc;
-use vello::{AaConfig, AaSupport, RenderParams, Renderer, RendererOptions, Scene};
+use vello::{AaConfig, AaSupport, DebugLayers, RenderParams, Renderer, RendererOptions, Scene};
 use wgpu::util::DeviceExt;
-use wgpu::{Buffer, BufferDescriptor, ShaderModule, SurfaceConfiguration, SurfaceError, Texture, TextureView};
+use wgpu::{Buffer, BufferDescriptor, Origin3d, ShaderModule, SurfaceConfiguration, SurfaceError, Texture, TextureAspect, TextureView};
 
 #[cfg(target_arch = "wasm32")]
 use web_sys::HtmlCanvasElement;
@@ -129,12 +128,14 @@ unsafe impl StaticType for Surface {
 	type Static = Surface;
 }
 
+pub use graphene_core::renderer::RenderContext;
+
 // pub trait SpirVCompiler {
 // 	fn compile(&self, network: &[ProtoNetwork], io: &ShaderIO) -> Result<Shader>;
 // }
 
 impl WgpuExecutor {
-	pub async fn render_vello_scene(&self, scene: &Scene, surface: &WgpuSurface, width: u32, height: u32) -> Result<()> {
+	pub async fn render_vello_scene(&self, scene: &Scene, surface: &WgpuSurface, width: u32, height: u32, context: &RenderContext) -> Result<()> {
 		let surface = &surface.surface.inner;
 		let surface_caps = surface.get_capabilities(&self.context.adapter);
 		surface.configure(
@@ -159,10 +160,23 @@ impl WgpuExecutor {
 			width,
 			height,
 			antialiasing_method: AaConfig::Msaa8,
+			debug: DebugLayers::all(),
 		};
 
 		{
 			let mut renderer = self.vello_renderer.lock().unwrap();
+			for (id, texture) in context.ressource_overrides.iter() {
+				let texture_view = wgpu::ImageCopyTextureBase {
+					texture: texture.clone(),
+					mip_level: 0,
+					origin: Origin3d::ZERO,
+					aspect: TextureAspect::All,
+				};
+				renderer.override_image(
+					&vello::peniko::Image::new(vello::peniko::Blob::from_raw_parts(Arc::new(vec![]), *id), vello::peniko::Format::Rgba8, 0, 0),
+					Some(texture_view),
+				);
+			}
 			renderer
 				.render_to_surface_async(&self.context.device, &self.context.queue, scene, &surface_texture, &render_params)
 				.await
@@ -229,14 +243,14 @@ impl WgpuExecutor {
 		let bytes = data.to_bytes();
 		let usage = match options {
 			TextureBufferOptions::Storage => wgpu::TextureUsages::STORAGE_BINDING | wgpu::TextureUsages::COPY_DST | wgpu::TextureUsages::COPY_SRC,
-			TextureBufferOptions::Texture => wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
+			TextureBufferOptions::Texture => wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST | wgpu::TextureUsages::COPY_SRC,
 			TextureBufferOptions::Surface => wgpu::TextureUsages::RENDER_ATTACHMENT,
 		};
 
 		let format = match T::format() {
 			TextureBufferType::Rgba32Float => wgpu::TextureFormat::Rgba32Float,
 			TextureBufferType::Rgba16Float => wgpu::TextureFormat::Rgba16Float,
-			TextureBufferType::Rgba8Srgb => wgpu::TextureFormat::Bgra8UnormSrgb,
+			TextureBufferType::Rgba8Srgb => wgpu::TextureFormat::Rgba8UnormSrgb,
 		};
 
 		let buffer = self.context.device.create_texture_with_data(
@@ -288,6 +302,7 @@ impl WgpuExecutor {
 			module: &layout.shader.0,
 			entry_point: layout.entry_point.as_str(),
 			compilation_options: Default::default(),
+			cache: None,
 		});
 		let bind_group_layout = compute_pipeline.get_bind_group_layout(0);
 
@@ -625,6 +640,7 @@ impl WgpuExecutor {
 			// If the pipeline will be used with a multiview render pass, this
 			// indicates how many array layers the attachments will have.
 			multiview: None,
+			cache: None,
 		});
 
 		let vertex_buffer = context.device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
@@ -949,8 +965,9 @@ pub struct UploadTextureNode<Executor> {
 }
 
 #[node_macro::node_fn(UploadTextureNode)]
-async fn upload_texture<'a: 'input>(input: ImageFrame<Color>, executor: &'a WgpuExecutor) -> ShaderInputFrame {
-	let new_data: Vec<RGBA16F> = input.image.data.into_iter().map(|c| c.into()).collect();
+async fn upload_texture<'a: 'input>(input: ImageFrame<Color>, executor: &'a WgpuExecutor) -> TextureFrame {
+	// let new_data: Vec<RGBA16F> = input.image.data.into_iter().map(|c| c.into()).collect();
+	let new_data = input.image.data.into_iter().map(|c| SRGBA8::from(c)).collect();
 	let new_image = Image {
 		width: input.image.width,
 		height: input.image.height,
@@ -959,9 +976,14 @@ async fn upload_texture<'a: 'input>(input: ImageFrame<Color>, executor: &'a Wgpu
 	};
 
 	let shader_input = executor.create_texture_buffer(new_image, TextureBufferOptions::Texture).unwrap();
+	let texture = match shader_input {
+		ShaderInput::TextureBuffer(buffer, _) => buffer,
+		ShaderInput::StorageTextureBuffer(buffer, _) => buffer,
+		_ => unreachable!("Unsupported ShaderInput type"),
+	};
 
-	ShaderInputFrame {
-		shader_input: Arc::new(shader_input),
+	TextureFrame {
+		texture: texture.into(),
 		transform: input.transform,
 	}
 }