From ab2ef99671e545405b24c7a668f18ab62c005486 Mon Sep 17 00:00:00 2001 From: Tad Hardesty Date: Thu, 26 Aug 2021 20:33:30 -0700 Subject: [PATCH] Add `dmm-tools render-many` subcommand to speed up MapDiffBot --- src/cli/main.rs | 177 ++++++++++++++++++++++++++++++++- src/tools/minimap.rs | 2 +- src/tools/render_passes/mod.rs | 12 ++- 3 files changed, 181 insertions(+), 10 deletions(-) diff --git a/src/cli/main.rs b/src/cli/main.rs index 839c7d2d..efb7de2b 100644 --- a/src/cli/main.rs +++ b/src/cli/main.rs @@ -14,11 +14,13 @@ extern crate dmm_tools; use std::collections::HashMap; use std::fmt; -use std::path::Path; +use std::io::Write; +use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicIsize, Ordering}; use std::sync::RwLock; use std::collections::HashSet; +use rayon::iter::{IndexedParallelIterator, IntoParallelIterator, ParallelIterator}; use structopt::StructOpt; use dm::objtree::ObjectTree; @@ -67,7 +69,7 @@ impl Context { _ => dm::DEFAULT_ENV.into(), }, }; - println!("parsing {}", environment.display()); + eprintln!("parsing {}", environment.display()); if let Some(parent) = environment.parent() { self.icon_cache.set_icons_root(&parent); @@ -175,6 +177,8 @@ enum Command { /// The list of maps to show info on. files: Vec, }, + /// Read a JSON RenderManyCommand from stdin, execute it, and print a RenderManyCommandResult. + RenderMany, } fn run(opt: &Opt, command: &Command, context: &mut Context) { @@ -329,7 +333,6 @@ fn run(opt: &Opt, command: &Command, context: &mut Context) { }; if parallel { - use rayon::iter::{IntoParallelIterator, ParallelIterator}; ((min.z - 1)..(max.z)).into_par_iter().for_each(do_z_level); } else { ((min.z - 1)..(max.z)).into_iter().for_each(do_z_level); @@ -337,7 +340,6 @@ fn run(opt: &Opt, command: &Command, context: &mut Context) { }; if parallel { - use rayon::iter::{IntoParallelIterator, ParallelIterator}; // Suboptimal due to mixing I/O in with what's meant for CPU // tasks, but it should get the job done for now. paths.into_par_iter().for_each(perform_job); @@ -404,16 +406,26 @@ fn run(opt: &Opt, command: &Command, context: &mut Context) { output_json(&report); }, // -------------------------------------------------------------------- + Command::RenderMany => { + let stdin = std::io::stdin(); + let command: RenderManyCommand = serde_json::from_reader(stdin.lock()).unwrap(); + context.objtree(opt); + let result = render_many(context, command); + let stdout = std::io::stdout(); + serde_json::to_writer(stdout.lock(), &result).unwrap(); + } + // -------------------------------------------------------------------- } } // ---------------------------------------------------------------------------- // Argument parsing helpers -#[derive(Debug, Copy, Clone)] +#[derive(Debug, Copy, Clone, Deserialize)] struct CoordArg { x: usize, y: usize, + #[serde(default)] z: usize, } @@ -468,3 +480,158 @@ fn output_json(t: &T) { serde_json::to_writer(&mut stdout, t).unwrap(); stdout.write_all(b"\n").unwrap(); } + +// ---------------------------------------------------------------------------- +// Render Many implementation + +#[derive(Deserialize)] +struct RenderManyCommand { + /// Directory to write results to. + output_directory: PathBuf, + + /// `.dmm` files to render. + /// + /// Including a file multiple times in this list is less efficient than + /// combining its chunks into one entry in this list. + files: Vec, + + /// Render passes to enable, or `["all"]` for all not explicitly disabled. + #[serde(default)] + enable: Vec, + + /// Render passes to disable, or `["all"]` for all not explicitly enabled. + #[serde(default)] + disable: Vec, +} + +#[derive(Deserialize)] +struct RenderManyFile { + /// The path to the `.dmm` file to render. + path: PathBuf, + /// Defaults to one chunk per z-level if omitted. + chunks: Option>, +} + +#[derive(Deserialize, Debug)] +struct RenderManyChunk { + /// The z-level to render. + z: usize, + /// Defaults to 1. + min_x: Option, + /// Defaults to 1. + min_y: Option, + /// Defaults to the X size of the map. + max_x: Option, + /// Defaults to the Y size of the map. + max_y: Option, +} + +#[derive(Serialize)] +struct RenderManyCommandResult { + /// Results for each file in the same order as the input. + files: Vec, +} + +#[derive(Serialize)] +struct RenderManyFileResult { + /// Results for each chunk in the same order as the input. + /// + /// If `chunks` was omitted in the input, this has one chunk per z-level. + chunks: Vec, +} + +#[derive(Serialize)] +struct RenderManyChunkResult { + /// The filename, not including the output directory, that the chunk was written to. + filename: PathBuf, +} + +fn render_many(context: &Context, command: RenderManyCommand) -> RenderManyCommandResult { + // Parse the object tree and otherwise prepare the context. + if context + .dm_context + .errors() + .iter() + .filter(|e| e.severity() <= dm::Severity::Error) + .next() + .is_some() + { + eprintln!("there were some parsing errors; render may be inaccurate") + } + let Context { + ref objtree, + ref icon_cache, + ref exit_status, + .. + } = *context; + let render_passes = &dmm_tools::render_passes::configure_list(&context.dm_context.config().map_renderer, &command.enable, &command.disable); + let errors: RwLock> = Default::default(); + + // Prepare output directory. + let output_directory = command.output_directory; + if let Err(e) = std::fs::create_dir_all(&output_directory) { + eprintln!("failed to create output directory {}:\n{}", output_directory.display(), e); + exit_status.fetch_add(1, Ordering::Relaxed); + panic!(); + } + + + // Iterate over the maps + let result_files: Vec<_> = command.files.into_par_iter().enumerate().map(|(file_idx, file)| { + eprintln!("{}: load {}", file_idx, file.path.display()); + let stem = file.path.file_stem().unwrap().to_string_lossy(); + let map = dmm::Map::from_file(&file.path).unwrap(); // TODO: error handling + let (dim_x, dim_y, dim_z) = map.dim_xyz(); + + // If `chunks` was not specified, render one chunk per z-level. + let chunks = file.chunks.unwrap_or_else(|| (1..=dim_z).map(|z| RenderManyChunk { + z, + min_x: None, + min_y: None, + max_x: None, + max_y: None, + }).collect()); + + let result_chunks: Vec<_> = chunks.into_par_iter().enumerate().map(|(chunk_idx, chunk)| { + eprintln!("{}/{}: render {:?}", file_idx, chunk_idx, chunk); + + // Render the image. + let bump = Default::default(); + let minimap_context = minimap::Context { + objtree, + map: &map, + level: map.z_level(chunk.z - 1), + // Default and clamp to [1, max]. + min: (chunk.min_x.unwrap_or(1).max(1) - 1, chunk.min_y.unwrap_or(1).max(1) - 1), + max: (chunk.max_x.unwrap_or(dim_x).min(dim_x) - 1, chunk.max_y.unwrap_or(dim_y).min(dim_y) - 1), + render_passes, + errors: &errors, + bump: &bump, + }; + let image = minimap::generate(minimap_context, icon_cache).unwrap(); // TODO: error handling + + // Write it to file. + let filename = PathBuf::from(format!( + "{}_z{}_chunk{}.png", + stem, + chunk.z, + chunk_idx, + )); + eprintln!("{}/{}: save {}", file_idx, chunk_idx, filename.display()); + let outfile = output_directory.join(&filename); + image.to_file(&outfile).unwrap(); // TODO: error handling + + RenderManyChunkResult { + filename, + } + }).collect(); + + RenderManyFileResult { + chunks: result_chunks, + } + }).collect(); + + RenderManyCommandResult { + files: result_files, + } +} diff --git a/src/tools/minimap.rs b/src/tools/minimap.rs index 589998e4..c02540f9 100644 --- a/src/tools/minimap.rs +++ b/src/tools/minimap.rs @@ -151,7 +151,7 @@ pub fn generate(ctx: Context, icon_cache: &IconCache) -> Result { } else { let key = format!("bad icon: {:?}, state: {:?}", sprite.icon, sprite.icon_state); if !ctx.errors.read().unwrap().contains(&key) { - println!("{}", key); + eprintln!("{}", key); ctx.errors.write().unwrap().insert(key); } } diff --git a/src/tools/render_passes/mod.rs b/src/tools/render_passes/mod.rs index 031c6489..97e33be1 100644 --- a/src/tools/render_passes/mod.rs +++ b/src/tools/render_passes/mod.rs @@ -121,14 +121,18 @@ pub const RENDER_PASSES: &[RenderPassInfo] = &[ pub fn configure(renderer_config: &dm::config::MapRenderer, include: &str, exclude: &str) -> Vec> { let include: Vec<&str> = include.split(",").collect(); let exclude: Vec<&str> = exclude.split(",").collect(); - let include_all = include.iter().any(|&name| name == "all"); - let exclude_all = exclude.iter().any(|&name| name == "all"); + configure_list(renderer_config, &include, &exclude) +} + +pub fn configure_list>(renderer_config: &dm::config::MapRenderer, include: &[T], exclude: &[T]) -> Vec> { + let include_all = include.iter().any(|name| name.as_ref() == "all"); + let exclude_all = exclude.iter().any(|name| name.as_ref() == "all"); let mut output = Vec::new(); for pass in RENDER_PASSES { - let included = if include.iter().any(|&name| name == pass.name) { + let included = if include.iter().any(|name| name.as_ref() == pass.name) { true - } else if exclude.iter().any(|&name| name == pass.name) { + } else if exclude.iter().any(|name| name.as_ref() == pass.name) { false } else if include_all { true