Integrate Stable Diffusion with the Imaginate layer (#784)

* Add AI Artist layer

* WIP add a button to download the rendered folder under an AI Artist layer

* Successfully download the correct image

* Break out image downloading JS into helper function

* Change file download from using data URLs to blob URLs

* WIP rasterize to blob

* Remove dimensions from AI Artist layer

* Successfully draw rasterized image on layer after calculation

* Working txt2img generation based on user prompt

* Add img2img and the main parameters

* Fix ability to rasterize multi-depth documents with blob URL images by switching them to base64

* Fix test

* Rasterize with artboard background color

* Allow aspect ratio stretch of AI Artist images

* Add automatic resolution choosing

* Add a terminate button, and make the lifecycle more robust

* Add negative prompt

* Add range bounds for parameter inputs

* Add seed

* Add tiling and restore faces

* Add server status check, server hostname customization, and resizing layer to fit AI Artist resolution

* Fix background color of infinite canvas rasterization

* Escape prompt text sent in the JSON

* Revoke blob URLs when cleared/replaced to reduce memory leak

* Fix welcome screen logo color

* Add PreferencesMessageHandler

* Add persistent storage of preferences

* Fix crash introduced in previous commit when moving mouse on page load

* Add tooltips to the AI Artist layer properties

* Integrate AI Artist tool into the raster section of the tool shelf

* Add a refresh button to the connection status

* Fix crash when generating and switching to a different document tab

* Add persistent image storage to AI Artist layers and fix duplication bugs

* Add a generate with random seed button

* Simplify and standardize message names

* Majorly improve robustness of networking code

* Fix race condition causing default server hostname to show disconnected when app loads with AI Artist layer selected (probably, not confirmed fixed)

* Clean up messages and function calls by changing arguments into structs

* Update API to more recent server commit

* Add support for picking the sampling method

* Add machinery for filtering selected layers with type

* Replace placeholder button icons

* Improve the random icon by tilting the dice

* Use selected_layers() instead of repeating that code

* Fix borrow error

* Change message flow in progress towards fixing #797

* Allow loading image on non-active document (fixes #797)

* Reduce code duplication with rasterization

* Add AI Artist tool and layer icons, and remove ugly node layer icon style

* Rename "AI Artist" codename to "Imaginate" feature name

Co-authored-by: otdavies <oliver@psyfer.io>
Co-authored-by: 0hypercube <0hypercube@gmail.com>

frontend/wasm/src/editor_api.rs

@@ -10,11 +10,11 @@ use editor::application::Editor;
 use editor::consts::{FILE_SAVE_SUFFIX, GRAPHITE_DOCUMENT_VERSION};
 use editor::messages::input_mapper::utility_types::input_keyboard::ModifierKeys;
 use editor::messages::input_mapper::utility_types::input_mouse::{EditorMouseState, ScrollDelta, ViewportBounds};
-use editor::messages::portfolio::document::utility_types::misc::Platform;
+use editor::messages::portfolio::utility_types::{ImaginateServerStatus, Platform};
 use editor::messages::prelude::*;
 use graphene::color::Color;
+use graphene::layers::imaginate_layer::ImaginateStatus;
 use graphene::LayerId;
-use graphene::Operation;
 
 use serde::Serialize;
 use serde_wasm_bindgen::{self, from_value};
@@ -168,6 +168,13 @@ impl JsEditorHandle {
 		}
 	}
 
+	#[wasm_bindgen(js_name = loadPreferences)]
+	pub fn load_preferences(&self, preferences: String) {
+		let message = PreferencesMessage::Load { preferences };
+
+		self.dispatch(message);
+	}
+
 	#[wasm_bindgen(js_name = selectDocument)]
 	pub fn select_document(&self, document_id: u64) {
 		let message = PortfolioMessage::SelectDocument { document_id };
@@ -427,11 +434,71 @@ impl JsEditorHandle {
 		self.dispatch(message);
 	}
 
-	/// Sends the blob url generated by js
-	#[wasm_bindgen(js_name = setImageBlobUrl)]
-	pub fn set_image_blob_url(&self, path: Vec<LayerId>, blob_url: String, width: f64, height: f64) {
-		let dimensions = (width, height);
-		let message = Operation::SetImageBlobUrl { path, blob_url, dimensions };
+	/// Sends the blob URL generated by JS to the Image layer
+	#[wasm_bindgen(js_name = setImageBlobURL)]
+	pub fn set_image_blob_url(&self, document_id: u64, layer_path: Vec<LayerId>, blob_url: String, width: f64, height: f64) {
+		let resolution = (width, height);
+		let message = PortfolioMessage::SetImageBlobUrl {
+			document_id,
+			layer_path,
+			blob_url,
+			resolution,
+		};
+		self.dispatch(message);
+	}
+
+	/// Sends the blob URL generated by JS to the Imaginate layer in the respective document
+	#[wasm_bindgen(js_name = setImaginateImageData)]
+	pub fn set_imaginate_image_data(&self, document_id: u64, layer_path: Vec<LayerId>, image_data: Vec<u8>) {
+		let message = PortfolioMessage::ImaginateSetImageData { document_id, layer_path, image_data };
+		self.dispatch(message);
+	}
+
+	/// Sends the blob URL generated by JS to the Imaginate layer in the respective document
+	#[wasm_bindgen(js_name = setImaginateBlobURL)]
+	pub fn set_imaginate_blob_url(&self, document_id: u64, layer_path: Vec<LayerId>, blob_url: String, width: f64, height: f64) {
+		let resolution = (width, height);
+		let message = PortfolioMessage::ImaginateSetBlobUrl {
+			document_id,
+			layer_path,
+			blob_url,
+			resolution,
+		};
+		self.dispatch(message);
+	}
+
+	/// Notifies the Imaginate layer of a new percentage of completion and whether or not it's currently generating
+	#[wasm_bindgen(js_name = setImaginateGeneratingStatus)]
+	pub fn set_imaginate_generating_status(&self, document_id: u64, path: Vec<LayerId>, percent: Option<f64>, status: String) {
+		let status = match status.as_str() {
+			"Idle" => ImaginateStatus::Idle,
+			"Beginning" => ImaginateStatus::Beginning,
+			"Uploading" => ImaginateStatus::Uploading(percent.expect("Percent needs to be supplied to set ImaginateStatus::Uploading")),
+			"Generating" => ImaginateStatus::Generating,
+			"Terminating" => ImaginateStatus::Terminating,
+			"Terminated" => ImaginateStatus::Terminated,
+			_ => panic!("Invalid string from JS for ImaginateStatus, received: {}", status),
+		};
+
+		let percent = if matches!(status, ImaginateStatus::Uploading(_)) { None } else { percent };
+
+		let message = PortfolioMessage::ImaginateSetGeneratingStatus { document_id, path, percent, status };
+		self.dispatch(message);
+	}
+
+	/// Notifies the editor that the Imaginate server is available or unavailable
+	#[wasm_bindgen(js_name = setImaginateServerStatus)]
+	pub fn set_imaginate_server_status(&self, available: bool) {
+		let message: Message = match available {
+			true => PortfolioMessage::ImaginateSetServerStatus {
+				status: ImaginateServerStatus::Connected,
+			}
+			.into(),
+			false => PortfolioMessage::ImaginateSetServerStatus {
+				status: ImaginateServerStatus::Unavailable,
+			}
+			.into(),
+		};
 		self.dispatch(message);
 	}