Add drag-and-drop and copy-paste file importing/opening throughout the UI (#2012)

* Add file importing by dragging and dropping throughout the UI

* Disable comment-profiling-changes.yaml

* Fix CI

frontend/src/components/layout/LayoutCol.svelte

@@ -42,6 +42,7 @@
 	on:dragleave
 	on:dragover
 	on:dragstart
+	on:drop
 	on:mouseup
 	on:pointerdown
 	on:pointerenter
@@ -58,7 +59,6 @@ on:copy
 on:cut
 on:drag
 on:dragenter
-on:drop
 on:focus
 on:fullscreenchange
 on:fullscreenerror

frontend/src/components/layout/LayoutRow.svelte

@@ -42,6 +42,7 @@
 	on:dragleave
 	on:dragover
 	on:dragstart
+	on:drop
 	on:mouseup
 	on:pointerdown
 	on:pointerenter
@@ -58,7 +59,6 @@ on:copy
 on:cut
 on:drag
 on:dragenter
-on:drop
 on:focus
 on:fullscreenchange
 on:fullscreenerror

frontend/src/components/panels/Document.svelte

@@ -118,23 +118,33 @@
 		};
 	})($document.toolShelfLayout.layout[0]);
 
-	function pasteFile(e: DragEvent) {
+	function dropFile(e: DragEvent) {
 		const { dataTransfer } = e;
+		const [x, y] = e.target instanceof Element && e.target.closest("[data-viewport]") ? [e.clientX, e.clientY] : [undefined, undefined];
 		if (!dataTransfer) return;
+
 		e.preventDefault();
 
 		Array.from(dataTransfer.items).forEach(async (item) => {
 			const file = item.getAsFile();
-			if (file?.type.includes("svg")) {
-				const svgData = await file.text();
-				editor.handle.pasteSvg(svgData, e.clientX, e.clientY);
+			if (!file) return;
 
+			if (file.type.includes("svg")) {
+				const svgData = await file.text();
+				editor.handle.pasteSvg(file.name, svgData, x, y);
 				return;
 			}
 
-			if (file?.type.startsWith("image")) {
+			if (file.type.startsWith("image")) {
 				const imageData = await extractPixelData(file);
-				editor.handle.pasteImage(new Uint8Array(imageData.data), imageData.width, imageData.height, e.clientX, e.clientY);
+				editor.handle.pasteImage(file.name, new Uint8Array(imageData.data), imageData.width, imageData.height, x, y);
+				return;
+			}
+
+			if (file.name.endsWith(".graphite")) {
+				const content = await file.text();
+				editor.handle.openDocumentFile(file.name, content);
+				return;
 			}
 		});
 	}
@@ -426,7 +436,7 @@
 	});
 </script>
 
-<LayoutCol class="document">
+<LayoutCol class="document" on:dragover={(e) => e.preventDefault()} on:drop={dropFile}>
 	<LayoutRow class="options-bar" classes={{ "for-graph": $document.graphViewOverlayOpen }} scrollableX={true}>
 		{#if !$document.graphViewOverlayOpen}
 			<WidgetLayout layout={$document.documentModeLayout} />
@@ -482,7 +492,7 @@
 							y={cursorTop}
 						/>
 					{/if}
-					<div class="viewport" on:pointerdown={(e) => canvasPointerDown(e)} on:dragover={(e) => e.preventDefault()} on:drop={(e) => pasteFile(e)} bind:this={viewport} data-viewport>
+					<div class="viewport" on:pointerdown={(e) => canvasPointerDown(e)} bind:this={viewport} data-viewport>
 						<svg class="artboards" style:width={canvasWidthCSS} style:height={canvasHeightCSS}>
 							{@html artworkSvg}
 						</svg>

frontend/src/components/panels/Layers.svelte

@@ -4,6 +4,7 @@
 	import { beginDraggingElement } from "@graphite/io-managers/drag";
 	import type { NodeGraphState } from "@graphite/state-providers/node-graph";
 	import { platformIsMac } from "@graphite/utility-functions/platform";
+	import { extractPixelData } from "@graphite/utility-functions/rasterization";
 	import type { Editor } from "@graphite/wasm-communication/editor";
 	import { defaultWidgetLayout, patchWidgetLayout, UpdateDocumentLayerDetails, UpdateDocumentLayerStructureJs, UpdateLayersPanelOptionsLayout } from "@graphite/wasm-communication/messages";
 	import type { DataBuffer, LayerPanelEntry } from "@graphite/wasm-communication/messages";
@@ -305,6 +306,8 @@
 	}
 
 	function updateInsertLine(event: DragEvent) {
+		if (!draggable) return;
+
 		// Stop the drag from being shown as cancelled
 		event.preventDefault();
 		dragInPanel = true;
@@ -312,13 +315,48 @@
 		if (list) draggingData = calculateDragIndex(list, event.clientY, draggingData?.select);
 	}
 
-	async function drop() {
-		if (draggingData && dragInPanel) {
-			const { select, insertParentId, insertIndex } = draggingData;
+	function drop(e: DragEvent) {
+		if (!draggingData) return;
+		const { select, insertParentId, insertIndex } = draggingData;
 
-			select?.();
-			editor.handle.moveLayerInTree(insertParentId, insertIndex);
+		e.preventDefault();
+
+		if (e.dataTransfer) {
+			// Moving layers
+			if (e.dataTransfer.items.length === 0) {
+				if (draggable && dragInPanel) {
+					select?.();
+					editor.handle.moveLayerInTree(insertParentId, insertIndex);
+				}
+			}
+			// Importing files
+			else {
+				Array.from(e.dataTransfer.items).forEach(async (item) => {
+					const file = item.getAsFile();
+					if (!file) return;
+
+					if (file.type.includes("svg")) {
+						const svgData = await file.text();
+						editor.handle.pasteSvg(file.name, svgData, undefined, undefined, insertParentId, insertIndex);
+						return;
+					}
+
+					if (file.type.startsWith("image")) {
+						const imageData = await extractPixelData(file);
+						editor.handle.pasteImage(file.name, new Uint8Array(imageData.data), imageData.width, imageData.height, undefined, undefined, insertParentId, insertIndex);
+						return;
+					}
+
+					// When we eventually have sub-documents, this should be changed to import the document instead of opening it in a separate tab
+					if (file.name.endsWith(".graphite")) {
+						const content = await file.text();
+						editor.handle.openDocumentFile(file.name, content);
+						return;
+					}
+				});
+			}
 		}
+
 		draggingData = undefined;
 		fakeHighlight = undefined;
 		dragInPanel = false;
@@ -369,7 +407,7 @@
 		<WidgetLayout layout={layersPanelOptionsLayout} />
 	</LayoutRow>
 	<LayoutRow class="list-area" scrollableY={true}>
-		<LayoutCol class="list" data-layer-panel bind:this={list} on:click={() => deselectAllLayers()} on:dragover={(e) => draggable && updateInsertLine(e)} on:dragend={() => draggable && drop()}>
+		<LayoutCol class="list" data-layer-panel bind:this={list} on:click={() => deselectAllLayers()} on:dragover={updateInsertLine} on:dragend={drop} on:drop={drop}>
 			{#each layers as listing, index}
 				<LayoutRow
 					class="layer"

frontend/src/components/window/workspace/Panel.svelte

@@ -2,8 +2,6 @@
 	import Document from "@graphite/components/panels/Document.svelte";
 	import Layers from "@graphite/components/panels/Layers.svelte";
 	import Properties from "@graphite/components/panels/Properties.svelte";
-	import IconButton from "@graphite/components/widgets/buttons/IconButton.svelte";
-	import TextButton from "@graphite/components/widgets/buttons/TextButton.svelte";
 
 	const PANEL_COMPONENTS = {
 		Document,
@@ -18,11 +16,14 @@
 
 	import { platformIsMac, isEventSupported } from "@graphite/utility-functions/platform";
 
+	import { extractPixelData } from "@graphite/utility-functions/rasterization";
 	import type { Editor } from "@graphite/wasm-communication/editor";
 	import { type LayoutKeysGroup, type Key } from "@graphite/wasm-communication/messages";
 
 	import LayoutCol from "@graphite/components/layout/LayoutCol.svelte";
 	import LayoutRow from "@graphite/components/layout/LayoutRow.svelte";
+	import IconButton from "@graphite/components/widgets/buttons/IconButton.svelte";
+	import TextButton from "@graphite/components/widgets/buttons/TextButton.svelte";
 	import IconLabel from "@graphite/components/widgets/labels/IconLabel.svelte";
 	import TextLabel from "@graphite/components/widgets/labels/TextLabel.svelte";
 	import UserInputLabel from "@graphite/components/widgets/labels/UserInputLabel.svelte";
@@ -50,6 +51,35 @@
 		return reservedKey ? [CONTROL, ALT] : [CONTROL];
 	}
 
+	function dropFile(e: DragEvent) {
+		if (!e.dataTransfer) return;
+
+		e.preventDefault();
+
+		Array.from(e.dataTransfer.items).forEach(async (item) => {
+			const file = item.getAsFile();
+			if (!file) return;
+
+			if (file.type.includes("svg")) {
+				const svgData = await file.text();
+				editor.handle.pasteSvg(file.name, svgData);
+				return;
+			}
+
+			if (file.type.startsWith("image")) {
+				const imageData = await extractPixelData(file);
+				editor.handle.pasteImage(file.name, new Uint8Array(imageData.data), imageData.width, imageData.height);
+				return;
+			}
+
+			if (file.name.endsWith(".graphite")) {
+				const content = await file.text();
+				editor.handle.openDocumentFile(file.name, content);
+				return;
+			}
+		});
+	}
+
 	export async function scrollTabIntoView(newIndex: number) {
 		await tick();
 		tabElements[newIndex]?.div?.()?.scrollIntoView();
@@ -76,7 +106,7 @@
 						}
 					}}
 					on:mouseup={(e) => {
-						// Fallback for Safari:
+						// Middle mouse button click fallback for Safari:
 						// https://developer.mozilla.org/en-US/docs/Web/API/Element/auxclick_event#browser_compatibility
 						// The downside of using mouseup is that the mousedown didn't have to originate in the same element.
 						// A possible future improvement could save the target element during mousedown and check if it's the same here.
@@ -110,7 +140,7 @@
 		{#if panelType}
 			<svelte:component this={PANEL_COMPONENTS[panelType]} />
 		{:else}
-			<LayoutCol class="empty-panel">
+			<LayoutCol class="empty-panel" on:dragover={(e) => e.preventDefault()} on:drop={dropFile}>
 				<LayoutCol class="content">
 					<LayoutRow class="logotype">
 						<IconLabel icon="GraphiteLogotypeSolid" />

frontend/src/io-managers/drag.ts

@@ -11,9 +11,7 @@ export function createDragManager(): () => void {
 	// Return the destructor
 	return () => {
 		// We use setTimeout to sequence this drop after any potential users in the current call stack progression, since this will begin in an entirely new call stack later
-		setTimeout(() => {
-			document.removeEventListener("drop", clearDraggingElement);
-		}, 0);
+		setTimeout(() => document.removeEventListener("drop", clearDraggingElement), 0);
 	};
 }
 

frontend/src/io-managers/input.ts

@@ -283,17 +283,21 @@ export function createInputManager(editor: Editor, dialog: DialogState, portfoli
 			}
 
 			const file = item.getAsFile();
+			if (!file) return;
 
-			if (file?.type === "svg") {
+			if (file.type.includes("svg")) {
 				const text = await file.text();
-				editor.handle.pasteSvg(text);
-
+				editor.handle.pasteSvg(file.name, text);
 				return;
 			}
 
-			if (file?.type.startsWith("image")) {
+			if (file.type.startsWith("image")) {
 				const imageData = await extractPixelData(file);
-				editor.handle.pasteImage(new Uint8Array(imageData.data), imageData.width, imageData.height);
+				editor.handle.pasteImage(file.name, new Uint8Array(imageData.data), imageData.width, imageData.height);
+			}
+
+			if (file.name.endsWith(".graphite")) {
+				editor.handle.openDocumentFile(file.name, await file.text());
 			}
 		});
 	}
@@ -316,52 +320,63 @@ export function createInputManager(editor: Editor, dialog: DialogState, portfoli
 			if (!clipboardItems) throw new Error("Clipboard API unsupported");
 
 			// Read any layer data or images from the clipboard
-			Array.from(clipboardItems).forEach(async (item) => {
-				// Read plain text and, if it is a layer, pass it to the editor
-				if (item.types.includes("text/plain")) {
-					const blob = await item.getType("text/plain");
-					const reader = new FileReader();
-					reader.onload = () => {
-						const text = reader.result as string;
+			const success = await Promise.any(
+				Array.from(clipboardItems).map(async (item) => {
+					// Read plain text and, if it is a layer, pass it to the editor
+					if (item.types.includes("text/plain")) {
+						const blob = await item.getType("text/plain");
+						const reader = new FileReader();
+						reader.onload = () => {
+							const text = reader.result as string;
 
-						if (text.startsWith("graphite/layer: ")) {
-							editor.handle.pasteSerializedData(text.substring(16, text.length));
-						}
-					};
-					reader.readAsText(blob);
-				}
+							if (text.startsWith("graphite/layer: ")) {
+								editor.handle.pasteSerializedData(text.substring(16, text.length));
+							}
+						};
+						reader.readAsText(blob);
+						return true;
+					}
 
-				// Read an image from the clipboard and pass it to the editor to be loaded
-				const imageType = item.types.find((type) => type.startsWith("image/"));
+					// Read an image from the clipboard and pass it to the editor to be loaded
+					const imageType = item.types.find((type) => type.startsWith("image/"));
 
-				if (imageType === "svg") {
-					const blob = await item.getType("text/plain");
-					const reader = new FileReader();
-					reader.onload = () => {
-						const text = reader.result as string;
-						editor.handle.pasteSvg(text);
-					};
-					reader.readAsText(blob);
+					// Import the actual SVG content if it's an SVG
+					if (imageType?.includes("svg")) {
+						const blob = await item.getType("text/plain");
+						const reader = new FileReader();
+						reader.onload = () => {
+							const text = reader.result as string;
+							editor.handle.pasteSvg(undefined, text);
+						};
+						reader.readAsText(blob);
+						return true;
+					}
 
-					return;
-				}
+					// Import the bitmap image if it's an image
+					if (imageType) {
+						const blob = await item.getType(imageType);
+						const reader = new FileReader();
+						reader.onload = async () => {
+							if (reader.result instanceof ArrayBuffer) {
+								const imageData = await extractPixelData(new Blob([reader.result], { type: imageType }));
+								editor.handle.pasteImage(undefined, new Uint8Array(imageData.data), imageData.width, imageData.height);
+							}
+						};
+						reader.readAsArrayBuffer(blob);
+						return true;
+					}
 
-				if (imageType) {
-					const blob = await item.getType(imageType);
-					const reader = new FileReader();
-					reader.onload = async () => {
-						if (reader.result instanceof ArrayBuffer) {
-							const imageData = await extractPixelData(new Blob([reader.result], { type: imageType }));
-							editor.handle.pasteImage(new Uint8Array(imageData.data), imageData.width, imageData.height);
-						}
-					};
-					reader.readAsArrayBuffer(blob);
-				}
-			});
+					// The API limits what kinds of data we can access, so we can get copied images and our text encodings of copied nodes, but not files (like
+					// .graphite or even image files). However, the user can paste those with Ctrl+V, which we recommend they in the error message that's shown to them.
+					return false;
+				}),
+			);
+
+			if (!success) throw new Error("No valid clipboard data");
 		} catch (err) {
 			const unsupported = stripIndents`
 				This browser does not support reading from the clipboard.
-				Use the keyboard shortcut to paste instead.
+				Use the standard keyboard shortcut to paste instead.
 				`;
 			const denied = stripIndents`
 				The browser's clipboard permission has been denied.
@@ -369,11 +384,16 @@ export function createInputManager(editor: Editor, dialog: DialogState, portfoli
 				Open the browser's website settings (usually accessible
 				just left of the URL) to allow this permission.
 				`;
+			const nothing = stripIndents`
+				No valid clipboard data was found. You may have better
+				luck pasting with the standard keyboard shortcut instead.
+				`;
 
 			const matchMessage = {
 				"clipboard-read": unsupported,
 				"Clipboard API unsupported": unsupported,
 				"Permission denied": denied,
+				"No valid clipboard data": nothing,
 			};
 			const message = Object.entries(matchMessage).find(([key]) => String(err).includes(key))?.[1] || String(err);
 

frontend/src/state-providers/document.ts

@@ -92,11 +92,9 @@ export function createDocumentState(editor: Editor) {
 	});
 	editor.subscriptions.subscribeJsMessage(TriggerDelayedZoomCanvasToFitAll, () => {
 		// TODO: This is horribly hacky
-		setTimeout(() => editor.handle.zoomCanvasToFitAll(), 0);
-		setTimeout(() => editor.handle.zoomCanvasToFitAll(), 1);
-		setTimeout(() => editor.handle.zoomCanvasToFitAll(), 10);
-		setTimeout(() => editor.handle.zoomCanvasToFitAll(), 50);
-		setTimeout(() => editor.handle.zoomCanvasToFitAll(), 100);
+		[0, 1, 10, 50, 100, 200, 300, 400, 500].forEach((delay) => {
+			setTimeout(() => editor.handle.zoomCanvasToFitAll(), delay);
+		});
 	});
 
 	return {

frontend/src/state-providers/portfolio.ts

@@ -65,17 +65,22 @@ export function createPortfolioState(editor: Editor) {
 		editor.handle.openDocumentFile(data.filename, data.content);
 	});
 	editor.subscriptions.subscribeJsMessage(TriggerImport, async () => {
-		const data = await upload("image/*", "data");
+		const data = await upload("image/*", "both");
 
 		if (data.type.includes("svg")) {
-			const svg = new TextDecoder().decode(data.content);
-			editor.handle.pasteSvg(svg);
-
+			const svg = new TextDecoder().decode(data.content.data);
+			editor.handle.pasteSvg(data.filename, svg);
 			return;
 		}
 
-		const imageData = await extractPixelData(new Blob([data.content], { type: data.type }));
-		editor.handle.pasteImage(new Uint8Array(imageData.data), imageData.width, imageData.height);
+		// In case the user accidentally uploads a Graphite file, open it instead of failing to import it
+		if (data.filename.endsWith(".graphite")) {
+			editor.handle.openDocumentFile(data.filename, data.content.text);
+			return;
+		}
+
+		const imageData = await extractPixelData(new Blob([data.content.data], { type: data.type }));
+		editor.handle.pasteImage(data.filename, new Uint8Array(imageData.data), imageData.width, imageData.height);
 	});
 	editor.subscriptions.subscribeJsMessage(TriggerDownloadTextFile, (triggerFileDownload) => {
 		downloadFileText(triggerFileDownload.name, triggerFileDownload.document);

frontend/src/utility-functions/files.ts

@@ -22,7 +22,7 @@ export function downloadFileText(filename: string, text: string) {
 	downloadFileBlob(filename, blob);
 }
 
-export async function upload<T extends "text" | "data">(acceptedExtensions: string, textOrData: T): Promise<UploadResult<T>> {
+export async function upload<T extends "text" | "data" | "both">(acceptedExtensions: string, textOrData: T): Promise<UploadResult<T>> {
 	return new Promise<UploadResult<T>>((resolve, _) => {
 		const element = document.createElement("input");
 		element.type = "file";
@@ -36,7 +36,15 @@ export async function upload<T extends "text" | "data">(acceptedExtensions: stri
 
 					const filename = file.name;
 					const type = file.type;
-					const content = (textOrData === "text" ? await file.text() : new Uint8Array(await file.arrayBuffer())) as UploadResultType<T>;
+					const content = (
+						textOrData === "text"
+							? await file.text()
+							: textOrData === "data"
+								? new Uint8Array(await file.arrayBuffer())
+								: textOrData === "both"
+									? { text: await file.text(), data: new Uint8Array(await file.arrayBuffer()) }
+									: undefined
+					) as UploadResultType<T>;
 
 					resolve({ filename, type, content });
 				}
@@ -50,7 +58,7 @@ export async function upload<T extends "text" | "data">(acceptedExtensions: stri
 	});
 }
 export type UploadResult<T> = { filename: string; type: string; content: UploadResultType<T> };
-type UploadResultType<T> = T extends "text" ? string : T extends "data" ? Uint8Array : never;
+type UploadResultType<T> = T extends "text" ? string : T extends "data" ? Uint8Array : T extends "both" ? { text: string; data: Uint8Array } : never;
 
 export function blobToBase64(blob: Blob): Promise<string> {
 	return new Promise((resolve) => {

frontend/src/wasm-communication/messages.ts

@@ -129,7 +129,8 @@ export abstract class DocumentDetails {
 
 	readonly isSaved!: boolean;
 
-	readonly id!: bigint | string;
+	// This field must be provided by the subclass implementation
+	// readonly id!: bigint | string;
 
 	get displayName(): string {
 		return `${this.name}${this.isSaved ? "" : "*"}`;

frontend/tsconfig.json

@@ -1,7 +1,7 @@
 {
 	"compilerOptions": {
-		"target": "ES2020",
-		"module": "esnext",
+		"target": "ESNext",
+		"module": "ESNext",
 		"strict": true,
 		"importHelpers": true,
 		"moduleResolution": "node",
@@ -18,7 +18,7 @@
 			"@graphite-frontend/*": ["./*"],
 			"@graphite/*": ["src/*"]
 		},
-		"lib": ["esnext", "dom", "dom.iterable", "scripthost"]
+		"lib": ["ESNext", "DOM", "DOM.Iterable", "ScriptHost"]
 	},
 	"include": ["src/**/*.ts", "src/**/*.d.ts", "src/**/*.svelte", "*.ts", "*.js", "*.cjs"],
 	"exclude": ["node_modules"],

frontend/wasm/src/editor_api.rs

@@ -593,17 +593,55 @@ impl EditorHandle {
 
 	/// Pastes an image
 	#[wasm_bindgen(js_name = pasteImage)]
-	pub fn paste_image(&self, image_data: Vec<u8>, width: u32, height: u32, mouse_x: Option<f64>, mouse_y: Option<f64>) {
+	pub fn paste_image(
+		&self,
+		name: Option<String>,
+		image_data: Vec<u8>,
+		width: u32,
+		height: u32,
+		mouse_x: Option<f64>,
+		mouse_y: Option<f64>,
+		insert_parent_id: Option<u64>,
+		insert_index: Option<usize>,
+	) {
 		let mouse = mouse_x.and_then(|x| mouse_y.map(|y| (x, y)));
 		let image = graphene_core::raster::Image::from_image_data(&image_data, width, height);
-		let message = DocumentMessage::PasteImage { image, mouse };
+
+		let parent_and_insert_index = if let (Some(insert_parent_id), Some(insert_index)) = (insert_parent_id, insert_index) {
+			let insert_parent_id = NodeId(insert_parent_id);
+			let parent = LayerNodeIdentifier::new_unchecked(insert_parent_id);
+			Some((parent, insert_index))
+		} else {
+			None
+		};
+
+		let message = PortfolioMessage::PasteImage {
+			name,
+			image,
+			mouse,
+			parent_and_insert_index,
+		};
 		self.dispatch(message);
 	}
 
 	#[wasm_bindgen(js_name = pasteSvg)]
-	pub fn paste_svg(&self, svg: String, mouse_x: Option<f64>, mouse_y: Option<f64>) {
+	pub fn paste_svg(&self, name: Option<String>, svg: String, mouse_x: Option<f64>, mouse_y: Option<f64>, insert_parent_id: Option<u64>, insert_index: Option<usize>) {
 		let mouse = mouse_x.and_then(|x| mouse_y.map(|y| (x, y)));
-		let message = DocumentMessage::PasteSvg { svg, mouse };
+
+		let parent_and_insert_index = if let (Some(insert_parent_id), Some(insert_index)) = (insert_parent_id, insert_index) {
+			let insert_parent_id = NodeId(insert_parent_id);
+			let parent = LayerNodeIdentifier::new_unchecked(insert_parent_id);
+			Some((parent, insert_index))
+		} else {
+			None
+		};
+
+		let message = PortfolioMessage::PasteSvg {
+			name,
+			svg,
+			mouse,
+			parent_and_insert_index,
+		};
 		self.dispatch(message);
 	}