Add manually-runnable benchmarks for runtime profiling (#2005)

* Split benches into two files

* Implement executor update bench

* Restructure benchmarks

* Unify usages of wrap network in scope

* Remove unused imports

* Fix oom bug

* Remove bounding box impl
This commit is contained in:
Dennis Kobert 2024-09-25 10:52:41 +02:00 committed by GitHub
parent c5454af48b
commit f8c7ada572
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
25 changed files with 378 additions and 270 deletions

View file

@ -11,8 +11,7 @@ dealloc_nodes = ["graphene-core/dealloc_nodes"]
wgpu = []
tokio = ["dep:tokio"]
wayland = []
criterion = []
iai = []
loading = ["serde_json", "serde"]
[dependencies]
# Local dependencies
@ -40,6 +39,7 @@ wgpu-executor = { workspace = true }
# Optional workspace dependencies
serde = { workspace = true, optional = true }
tokio = { workspace = true, optional = true }
serde_json = { workspace = true, optional = true }
# Workspace dependencies
[target.'cfg(target_arch = "wasm32")'.dependencies]
@ -53,19 +53,17 @@ winit = { workspace = true }
[dev-dependencies]
# Workspace dependencies
serde_json = { workspace = true }
graph-craft = { workspace = true, features = ["serde"] }
graph-craft = { workspace = true, features = ["loading"] }
# Required dependencies
criterion = { version = "0.5", features = ["html_reports"]}
glob = "0.3"
iai-callgrind = { version = "0.12.3"}
# Benchmarks
[[bench]]
name = "compile_demo_art"
name = "compile_demo_art_criterion"
harness = false
# [[bench]]
# name = "exec_demo_art"
# harness = false
[[bench]]
name = "compile_demo_art_iai"
harness = false

View file

@ -1,64 +0,0 @@
use graph_craft::document::NodeNetwork;
#[cfg(any(feature = "criterion", feature = "iai"))]
use graph_craft::graphene_compiler::Compiler;
#[cfg(any(feature = "criterion", feature = "iai"))]
use graph_craft::proto::ProtoNetwork;
#[cfg(feature = "criterion")]
use criterion::{black_box, criterion_group, criterion_main, Criterion};
#[cfg(all(not(feature = "criterion"), feature = "iai"))]
use iai_callgrind::{black_box, library_benchmark, library_benchmark_group, main};
#[cfg(any(feature = "criterion", feature = "iai"))]
fn load_network(document_string: &str) -> NodeNetwork {
let document: serde_json::Value = serde_json::from_str(document_string).expect("Failed to parse document");
serde_json::from_value::<NodeNetwork>(document["network_interface"]["network"].clone()).expect("Failed to parse document")
}
#[cfg(any(feature = "criterion", feature = "iai"))]
fn compile(network: NodeNetwork) -> ProtoNetwork {
let compiler = Compiler {};
compiler.compile_single(network).unwrap()
}
#[cfg(all(not(feature = "criterion"), feature = "iai"))]
fn load_from_name(name: &str) -> NodeNetwork {
let content = std::fs::read(&format!("../../demo-artwork/{name}.graphite")).expect("failed to read file");
let network = load_network(std::str::from_utf8(&content).unwrap());
let content = std::str::from_utf8(&content).unwrap();
black_box(compile(black_box(network)));
load_network(content)
}
#[cfg(feature = "criterion")]
fn compile_to_proto(c: &mut Criterion) {
let artworks = glob::glob("../../demo-artwork/*.graphite").expect("failed to read glob pattern");
for path in artworks {
let Ok(path) = path else { continue };
let name = path.file_stem().unwrap().to_str().unwrap();
let content = std::fs::read(&path).expect("failed to read file");
let network = load_network(std::str::from_utf8(&content).unwrap());
c.bench_function(name, |b| b.iter_batched(|| network.clone(), |network| compile(black_box(network)), criterion::BatchSize::SmallInput));
}
}
#[cfg_attr(all(feature = "iai", not(feature = "criterion")), library_benchmark)]
#[cfg_attr(all(feature = "iai", not(feature="criterion")), benches::with_setup(args = ["isometric-fountain", "painted-dreams", "procedural-string-lights", "red-dress", "valley-of-spires"], setup = load_from_name))]
// Note that this can not be disabled with a `#[cfg(...)]` because this causes a compile error.
// Therefore negated condition is used in `#[cfg_attr(...)]` with the attribute `cfg(any())` that is always false.
pub fn iai_compile_to_proto(_input: NodeNetwork) {
#[cfg(all(feature = "iai", not(feature = "criterion")))]
black_box(compile(_input));
}
#[cfg(feature = "criterion")]
criterion_group!(benches, compile_to_proto);
#[cfg(feature = "criterion")]
criterion_main!(benches);
#[cfg(all(not(feature = "criterion"), feature = "iai"))]
library_benchmark_group!(name = compile_group; benchmarks = iai_compile_to_proto);
#[cfg(all(not(feature = "criterion"), feature = "iai"))]
main!(library_benchmark_groups = compile_group);
// An empty main function so the crate compiles with no features enabled.
#[cfg(all(not(feature = "criterion"), not(feature = "iai")))]
fn main() {}

View file

@ -0,0 +1,14 @@
use criterion::{black_box, criterion_group, criterion_main, Criterion};
use graph_craft::util::DEMO_ART;
fn compile_to_proto(c: &mut Criterion) {
use graph_craft::util::{compile, load_from_name};
let mut c = c.benchmark_group("Compile Network cold");
for name in DEMO_ART {
let network = load_from_name(name);
c.bench_function(name, |b| b.iter_batched(|| network.clone(), |network| compile(black_box(network)), criterion::BatchSize::SmallInput));
}
}
criterion_group!(benches, compile_to_proto);
criterion_main!(benches);

View file

@ -0,0 +1,13 @@
use graph_craft::document::NodeNetwork;
use graph_craft::util::*;
use iai_callgrind::{black_box, library_benchmark, library_benchmark_group, main};
#[library_benchmark]
#[benches::with_setup(args = ["isometric-fountain", "painted-dreams", "procedural-string-lights", "red-dress", "valley-of-spires"], setup = load_from_name)]
pub fn compile_to_proto(_input: NodeNetwork) {
black_box(compile(_input));
}
library_benchmark_group!(name = compile_group; benchmarks = compile_to_proto);
main!(library_benchmark_groups = compile_group);

View file

@ -26,20 +26,23 @@ fn merge_ids(a: NodeId, b: NodeId) -> NodeId {
/// Utility function for providing a default boolean value to serde.
#[inline(always)]
#[cfg(feature = "serde")]
fn return_true() -> bool {
true
}
// TODO: Eventually remove this (probably starting late 2024)
#[derive(Debug, serde::Deserialize)]
#[serde(untagged)]
#[derive(Debug)]
#[cfg_attr(feature = "serde", derive(serde::Deserialize))]
#[cfg_attr(feature = "serde", serde(untagged))]
enum NodeInputVersions {
OldNodeInput(OldNodeInput),
NodeInput(NodeInput),
}
// TODO: Eventually remove this (probably starting late 2024)
#[derive(Debug, serde::Deserialize)]
#[derive(Debug)]
#[cfg_attr(feature = "serde", derive(serde::Deserialize))]
pub enum OldNodeInput {
/// A reference to another node in the same network from which this node can receive its input.
Node { node_id: NodeId, output_index: usize, lambda: bool },
@ -56,11 +59,12 @@ pub enum OldNodeInput {
}
// TODO: Eventually remove this (probably starting late 2024)
use serde::Deserialize;
#[cfg(feature = "serde")]
fn deserialize_inputs<'de, D>(deserializer: D) -> Result<Vec<NodeInput>, D::Error>
where
D: serde::Deserializer<'de>,
{
use serde::Deserialize;
let input_versions = Vec::<NodeInputVersions>::deserialize(deserializer)?;
let inputs = input_versions
@ -95,7 +99,7 @@ pub struct DocumentNode {
///
/// In the root network, it is resolved when evaluating the borrow tree.
/// Ensure the click target in the encapsulating network is updated when the inputs cause the node shape to change (currently only when exposing/hiding an input) by using network.update_click_target(node_id).
#[serde(deserialize_with = "deserialize_inputs")]
#[cfg_attr(feature = "serde", serde(deserialize_with = "deserialize_inputs"))]
pub inputs: Vec<NodeInput>,
/// Manual composition is a way to override the default composition flow of one node into another.
///
@ -184,15 +188,15 @@ pub struct DocumentNode {
// A nested document network or a proto-node identifier.
pub implementation: DocumentNodeImplementation,
/// Represents the eye icon for hiding/showing the node in the graph UI. When hidden, a node gets replaced with an identity node during the graph flattening step.
#[serde(default = "return_true")]
#[cfg_attr(feature = "serde", serde(default = "return_true"))]
pub visible: bool,
/// When two different proto nodes hash to the same value (e.g. two value nodes each containing `2_u32` or two multiply nodes that have the same node IDs as input), the duplicates are removed.
/// See [`crate::proto::ProtoNetwork::generate_stable_node_ids`] for details.
/// However sometimes this is not desirable, for example in the case of a [`graphene_core::memo::MonitorNode`] that needs to be accessed outside of the graph.
#[serde(default)]
#[cfg_attr(feature = "serde", serde(default))]
pub skip_deduplication: bool,
/// The path to this node and its inputs and outputs as of when [`NodeNetwork::generate_node_paths`] was called.
#[serde(skip)]
#[cfg_attr(feature = "serde", serde(skip))]
pub original_location: OriginalLocation,
}
@ -480,7 +484,7 @@ pub enum OldDocumentNodeImplementation {
/// This describes a (document) node implemented as a proto node.
///
/// A proto node identifier which can be found in `node_registry.rs`.
#[serde(alias = "Unresolved")] // TODO: Eventually remove this alias (probably starting late 2024)
#[cfg_attr(feature = "serde", serde(alias = "Unresolved"))] // TODO: Eventually remove this alias (probably starting late 2024)
ProtoNode(ProtoNodeIdentifier),
/// The Extract variant is a tag which tells the compilation process to do something special. It invokes language-level functionality built for use by the ExtractNode to enable metaprogramming.
/// When the ExtractNode is compiled, it gets replaced by a value node containing a representation of the source code for the function/lambda of the document node that's fed into the ExtractNode
@ -515,7 +519,7 @@ pub enum DocumentNodeImplementation {
/// This describes a (document) node implemented as a proto node.
///
/// A proto node identifier which can be found in `node_registry.rs`.
#[serde(alias = "Unresolved")] // TODO: Eventually remove this alias (probably starting late 2024)
#[cfg_attr(feature = "serde", serde(alias = "Unresolved"))] // TODO: Eventually remove this alias (probably starting late 2024)
ProtoNode(ProtoNodeIdentifier),
/// The Extract variant is a tag which tells the compilation process to do something special. It invokes language-level functionality built for use by the ExtractNode to enable metaprogramming.
/// When the ExtractNode is compiled, it gets replaced by a value node containing a representation of the source code for the function/lambda of the document node that's fed into the ExtractNode
@ -573,25 +577,29 @@ impl DocumentNodeImplementation {
}
// TODO: Eventually remove this (probably starting late 2024)
#[derive(Debug, serde::Deserialize)]
#[serde(untagged)]
#[derive(Debug)]
#[cfg_attr(feature = "serde", derive(serde::Deserialize))]
#[cfg_attr(feature = "serde", serde(untagged))]
pub enum NodeExportVersions {
OldNodeInput(NodeOutput),
NodeInput(NodeInput),
}
// TODO: Eventually remove this (probably starting late 2024)
#[derive(Debug, serde::Deserialize)]
#[derive(Debug)]
#[cfg_attr(feature = "serde", derive(serde::Deserialize))]
pub struct NodeOutput {
pub node_id: NodeId,
pub node_output_index: usize,
}
// TODO: Eventually remove this (probably starting late 2024)
#[cfg(feature = "serde")]
fn deserialize_exports<'de, D>(deserializer: D) -> Result<Vec<NodeInput>, D::Error>
where
D: serde::Deserializer<'de>,
{
use serde::Deserialize;
let node_input_versions = Vec::<NodeExportVersions>::deserialize(deserializer)?;
// Convert Vec<NodeOutput> to Vec<NodeInput>
@ -617,11 +625,11 @@ where
pub struct OldDocumentNode {
/// A name chosen by the user for this instance of the node. Empty indicates no given name, in which case the node definition's name is displayed to the user in italics.
/// Ensure the click target in the encapsulating network is updated when this is modified by using network.update_click_target(node_id).
#[serde(default)]
#[cfg_attr(feature = "serde", serde(default))]
pub alias: String,
// TODO: Replace this name with a reference to the [`DocumentNodeDefinition`] node definition to use the name from there instead.
/// The name of the node definition, as originally set by [`DocumentNodeDefinition`], used to display in the UI and to display the appropriate properties.
#[serde(deserialize_with = "migrate_layer_to_merge")]
#[cfg_attr(feature = "serde", serde(deserialize_with = "migrate_layer_to_merge"))]
pub name: String,
/// The inputs to a node, which are either:
/// - From other nodes within this graph [`NodeInput::Node`],
@ -630,34 +638,34 @@ pub struct OldDocumentNode {
///
/// In the root network, it is resolved when evaluating the borrow tree.
/// Ensure the click target in the encapsulating network is updated when the inputs cause the node shape to change (currently only when exposing/hiding an input) by using network.update_click_target(node_id).
#[serde(deserialize_with = "deserialize_inputs")]
#[cfg_attr(feature = "serde", serde(deserialize_with = "deserialize_inputs"))]
pub inputs: Vec<NodeInput>,
pub manual_composition: Option<Type>,
// TODO: Remove once this references its definition instead (see above TODO).
/// Indicates to the UI if a primary output should be drawn for this node.
/// True for most nodes, but the Split Channels node is an example of a node that has multiple secondary outputs but no primary output.
#[serde(default = "return_true")]
#[cfg_attr(feature = "serde", serde(default = "return_true"))]
pub has_primary_output: bool,
// A nested document network or a proto-node identifier.
pub implementation: OldDocumentNodeImplementation,
/// User chosen state for displaying this as a left-to-right node or bottom-to-top layer. Ensure the click target in the encapsulating network is updated when the node changes to a layer by using network.update_click_target(node_id).
#[serde(default)]
#[cfg_attr(feature = "serde", serde(default))]
pub is_layer: bool,
/// Represents the eye icon for hiding/showing the node in the graph UI. When hidden, a node gets replaced with an identity node during the graph flattening step.
#[serde(default = "return_true")]
#[cfg_attr(feature = "serde", serde(default = "return_true"))]
pub visible: bool,
/// Represents the lock icon for locking/unlocking the node in the graph UI. When locked, a node cannot be moved in the graph UI.
#[serde(default)]
#[cfg_attr(feature = "serde", serde(default))]
pub locked: bool,
/// Metadata about the node including its position in the graph UI. Ensure the click target in the encapsulating network is updated when the node moves by using network.update_click_target(node_id).
pub metadata: OldDocumentNodeMetadata,
/// When two different proto nodes hash to the same value (e.g. two value nodes each containing `2_u32` or two multiply nodes that have the same node IDs as input), the duplicates are removed.
/// See [`crate::proto::ProtoNetwork::generate_stable_node_ids`] for details.
/// However sometimes this is not desirable, for example in the case of a [`graphene_core::memo::MonitorNode`] that needs to be accessed outside of the graph.
#[serde(default)]
#[cfg_attr(feature = "serde", serde(default))]
pub skip_deduplication: bool,
/// The path to this node and its inputs and outputs as of when [`NodeNetwork::generate_node_paths`] was called.
#[serde(skip)]
#[cfg_attr(feature = "serde", serde(skip))]
pub original_location: OriginalLocation,
}
@ -696,27 +704,28 @@ pub enum OldPreviewing {
pub struct OldNodeNetwork {
/// The list of data outputs that are exported from this network to the parent network.
/// Each export is a reference to a node within this network, paired with its output index, that is the source of the network's exported data.
#[serde(alias = "outputs", deserialize_with = "deserialize_exports")] // TODO: Eventually remove this alias (probably starting late 2024)
#[cfg_attr(feature = "serde", serde(alias = "outputs", deserialize_with = "deserialize_exports"))] // TODO: Eventually remove this alias (probably starting late 2024)
pub exports: Vec<NodeInput>,
/// The list of all nodes in this network.
//#[serde(serialize_with = "graphene_core::vector::serialize_hashmap", deserialize_with = "graphene_core::vector::deserialize_hashmap")]
//cfg_attr(feature = "serde", #[cfg_attr(feature = "serde", serde(serialize_with = "graphene_core::vector::serialize_hashmap", deserialize_with = "graphene_core::vector::deserialize_hashmap")))]
pub nodes: HashMap<NodeId, OldDocumentNode>,
/// Indicates whether the network is currently rendered with a particular node that is previewed, and if so, which connection should be restored when the preview ends.
#[serde(default)]
#[cfg_attr(feature = "serde", serde(default))]
pub previewing: OldPreviewing,
/// Temporary fields to store metadata for "Import"/"Export" UI-only nodes, eventually will be replaced with lines leading to edges
#[serde(default = "default_import_metadata")]
#[cfg_attr(feature = "serde", serde(default = "default_import_metadata"))]
pub imports_metadata: (NodeId, IVec2),
#[serde(default = "default_export_metadata")]
#[cfg_attr(feature = "serde", serde(default = "default_export_metadata"))]
pub exports_metadata: (NodeId, IVec2),
/// A network may expose nodes as constants which can by used by other nodes using a `NodeInput::Scope(key)`.
#[serde(default)]
//#[serde(serialize_with = "graphene_core::vector::serialize_hashmap", deserialize_with = "graphene_core::vector::deserialize_hashmap")]
#[cfg_attr(feature = "serde", serde(default))]
//cfg_attr(feature = "serde", #[cfg_attr(feature = "serde", serde(serialize_with = "graphene_core::vector::serialize_hashmap", deserialize_with = "graphene_core::vector::deserialize_hashmap")))]
pub scope_injections: HashMap<String, (NodeId, Type)>,
}
// TODO: Eventually remove this (probably starting late 2024)
#[cfg(feature = "serde")]
fn migrate_layer_to_merge<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result<String, D::Error> {
let mut s: String = serde::Deserialize::deserialize(deserializer)?;
if s == "Layer" {
@ -739,16 +748,22 @@ fn default_export_metadata() -> (NodeId, IVec2) {
pub struct NodeNetwork {
/// The list of data outputs that are exported from this network to the parent network.
/// Each export is a reference to a node within this network, paired with its output index, that is the source of the network's exported data.
#[serde(alias = "outputs", deserialize_with = "deserialize_exports")] // TODO: Eventually remove this alias (probably starting late 2024)
#[cfg_attr(feature = "serde", serde(alias = "outputs", deserialize_with = "deserialize_exports"))] // TODO: Eventually remove this alias (probably starting late 2024)
pub exports: Vec<NodeInput>,
/// TODO: Instead of storing import types in each NodeInput::Network connection, the types are stored here. This is similar to how types need to be defined for parameters when creating a function in Rust.
// pub import_types: Vec<Type>,
/// The list of all nodes in this network.
#[serde(serialize_with = "graphene_core::vector::serialize_hashmap", deserialize_with = "graphene_core::vector::deserialize_hashmap")]
#[cfg_attr(
feature = "serde",
serde(serialize_with = "graphene_core::vector::serialize_hashmap", deserialize_with = "graphene_core::vector::deserialize_hashmap")
)]
pub nodes: FxHashMap<NodeId, DocumentNode>,
/// A network may expose nodes as constants which can by used by other nodes using a `NodeInput::Scope(key)`.
#[serde(default)]
#[serde(serialize_with = "graphene_core::vector::serialize_hashmap", deserialize_with = "graphene_core::vector::deserialize_hashmap")]
#[cfg_attr(feature = "serde", serde(default))]
#[cfg_attr(
feature = "serde",
serde(serialize_with = "graphene_core::vector::serialize_hashmap", deserialize_with = "graphene_core::vector::deserialize_hashmap")
)]
pub scope_injections: FxHashMap<String, (NodeId, Type)>,
}

View file

@ -30,7 +30,7 @@ macro_rules! tagged_value {
$( $(#[$meta] ) *$identifier( $ty ), )*
RenderOutput(RenderOutput),
SurfaceFrame(graphene_core::SurfaceFrame),
#[serde(skip)]
#[cfg_attr(feature = "serde", serde(skip))]
EditorApi(Arc<WasmEditorApi>)
}
@ -117,7 +117,7 @@ tagged_value! {
String(String),
U32(u32),
U64(u64),
#[serde(alias = "F32")] // TODO: Eventually remove this alias (probably starting late 2024)
#[cfg_attr(feature = "serde", serde(alias = "F32"))] // TODO: Eventually remove this alias (probably starting late 2024)
F64(f64),
Bool(bool),
UVec2(UVec2),
@ -139,7 +139,7 @@ tagged_value! {
Fill(graphene_core::vector::style::Fill),
Stroke(graphene_core::vector::style::Stroke),
F64Array4([f64; 4]),
#[serde(alias = "VecF32")] // TODO: Eventually remove this alias (probably starting late 2024)
#[cfg_attr(feature = "serde", serde(alias = "VecF32"))] // TODO: Eventually remove this alias (probably starting late 2024)
VecF64(Vec<f64>),
VecU64(Vec<u64>),
NodePath(Vec<NodeId>),
@ -159,10 +159,10 @@ tagged_value! {
FillChoice(graphene_core::vector::style::FillChoice),
Gradient(graphene_core::vector::style::Gradient),
GradientType(graphene_core::vector::style::GradientType),
#[serde(alias = "GradientPositions")] // TODO: Eventually remove this alias (probably starting late 2024)
#[cfg_attr(feature = "serde", serde(alias = "GradientPositions"))] // TODO: Eventually remove this alias (probably starting late 2024)
GradientStops(graphene_core::vector::style::GradientStops),
OptionalColor(Option<graphene_core::raster::color::Color>),
#[serde(alias = "ManipulatorGroupIds")] // TODO: Eventually remove this alias (probably starting late 2024)
#[cfg_attr(feature = "serde", serde(alias = "ManipulatorGroupIds"))] // TODO: Eventually remove this alias (probably starting late 2024)
PointIds(Vec<graphene_core::vector::PointId>),
Font(graphene_core::text::Font),
BrushStrokes(Vec<graphene_core::vector::brush_stroke::BrushStroke>),

View file

@ -12,3 +12,6 @@ pub mod graphene_compiler;
pub mod imaginate_input;
pub mod wasm_application_io;
#[cfg(feature = "loading")]
pub mod util;

View file

@ -0,0 +1,21 @@
use crate::document::NodeNetwork;
use crate::graphene_compiler::Compiler;
use crate::proto::ProtoNetwork;
pub fn load_network(document_string: &str) -> NodeNetwork {
let document: serde_json::Value = serde_json::from_str(document_string).expect("Failed to parse document");
serde_json::from_value::<NodeNetwork>(document["network_interface"]["network"].clone()).expect("Failed to parse document")
}
pub fn compile(network: NodeNetwork) -> ProtoNetwork {
let compiler = Compiler {};
compiler.compile_single(network).unwrap()
}
pub fn load_from_name(name: &str) -> NodeNetwork {
let content = std::fs::read(format!("../../demo-artwork/{name}.graphite")).expect("failed to read file");
let content = std::str::from_utf8(&content).unwrap();
load_network(content)
}
pub static DEMO_ART: [&str; 5] = ["painted-dreams", "red-dress", "valley-of-spires", "isometric-fountain", "procedural-string-lights"];