mirror of
https://github.com/SpaceManiac/SpacemanDMM.git
synced 2025-12-23 05:36:47 +00:00
Add ability to run map rendering jobs in parallel
This commit is contained in:
Tad Hardesty
parent
8ad1e85673
commit
a33dedbdfc
3 changed files with 142 additions and 13 deletions
91
Cargo.lock
generated
91
Cargo.lock
generated
|
|
@ -27,6 +27,14 @@ dependencies = [
|
|||
"winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "arrayvec"
|
||||
version = "0.4.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "atty"
|
||||
version = "0.2.8"
|
||||
|
|
@ -127,6 +135,7 @@ version = "0.1.0"
|
|||
dependencies = [
|
||||
"dmm-tools 0.1.0",
|
||||
"dreammaker 0.1.0",
|
||||
"rayon 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_derive 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_json 1.0.10 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
|
|
@ -239,6 +248,37 @@ name = "cpp_utils"
|
|||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-deque"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"crossbeam-epoch 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dbghelp-sys"
|
||||
version = "0.2.0"
|
||||
|
|
@ -463,6 +503,11 @@ name = "lazy_static"
|
|||
version = "0.2.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "lazy_static"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.39"
|
||||
|
|
@ -526,6 +571,11 @@ dependencies = [
|
|||
"libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memoffset"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "ndarray"
|
||||
version = "0.11.1"
|
||||
|
|
@ -537,6 +587,11 @@ dependencies = [
|
|||
"num-traits 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nodrop"
|
||||
version = "0.1.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "num-complex"
|
||||
version = "0.1.42"
|
||||
|
|
@ -705,6 +760,27 @@ name = "rawpointer"
|
|||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"either 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rayon-core 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon-core"
|
||||
version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"crossbeam-deque 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.1.37"
|
||||
|
|
@ -753,6 +829,11 @@ dependencies = [
|
|||
"winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scopeguard"
|
||||
version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "0.9.15"
|
||||
|
|
@ -1153,6 +1234,7 @@ dependencies = [
|
|||
"checksum aho-corasick 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ca972c2ea5f742bfce5687b9aef75506a764f61d37f8f649047846a9686ddb66"
|
||||
"checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"
|
||||
"checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
|
||||
"checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"
|
||||
"checksum atty 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "af80143d6f7608d746df1520709e5d141c96f240b0e62b0aa41bdfb53374d9d4"
|
||||
"checksum backtrace 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "346d7644f0b5f9bc73082d3b2236b69a05fd35cce0cfa3724e184e6a5c9e2a2f"
|
||||
"checksum backtrace 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "ebbbf59b1c43eefa8c3ede390fcc36820b4999f7914104015be25025e0d62af2"
|
||||
|
|
@ -1174,6 +1256,9 @@ dependencies = [
|
|||
"checksum cpp_to_rust_build_tools 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0db0bdd5d0e531fc8c03b710a2dd8433b2dad26d7eb42363910bffd49a6cd292"
|
||||
"checksum cpp_to_rust_common 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "67af28b7b19500adbe79cd195cb10201d06dcf37e269d022ef59c4ede801b984"
|
||||
"checksum cpp_utils 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "74f3201ad4ca76504ae6c14cbbf1340c56e69f179be0200c474a9f0cc78a0b20"
|
||||
"checksum crossbeam-deque 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f739f8c5363aca78cfb059edf753d8f0d36908c348f3d8d1503f03d8b75d9cf3"
|
||||
"checksum crossbeam-epoch 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "59796cc6cbbdc6bb319161349db0c3250ec73ec7fcb763a51065ec4e2e158552"
|
||||
"checksum crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2760899e32a1d58d5abb31129f8fae5de75220bc2176e77ff7c627ae45c918d9"
|
||||
"checksum dbghelp-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "97590ba53bcb8ac28279161ca943a924d1fd4a8fb3fa63302591647c4fc5b850"
|
||||
"checksum deflate 0.7.17 (registry+https://github.com/rust-lang/crates.io-index)" = "4dddda59aaab719767ab11d3efd9a714e95b610c4445d4435765021e9d52dfb1"
|
||||
"checksum dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "09c3753c3db574d215cba4ea76018483895d7bff25a31b49ba45db21c48e50ab"
|
||||
|
|
@ -1198,6 +1283,7 @@ dependencies = [
|
|||
"checksum khronos_api 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d9ef23fcc4059260c5936f638c9805ebfc87cb172fa6661d130cba7f97d58f55"
|
||||
"checksum languageserver-types 0.30.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1541f9b22687f060511d213036e1f058797c48e3501e177f01cb6e88de802f5b"
|
||||
"checksum lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73"
|
||||
"checksum lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c8f31047daa365f19be14b47c29df4f7c3b581832407daabe6ae77397619237d"
|
||||
"checksum libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)" = "f54263ad99207254cf58b5f701ecb432c717445ea2ee8af387334bdd1a03fdff"
|
||||
"checksum linked-hash-map 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "70fb39025bc7cdd76305867c4eccf2f2dcf6e9a57f5b21a93e1c2d86cd03ec9e"
|
||||
"checksum log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b"
|
||||
|
|
@ -1207,7 +1293,9 @@ dependencies = [
|
|||
"checksum memchr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d8b629fb514376c675b98c1421e80b151d3817ac42d7c667717d282761418d20"
|
||||
"checksum memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "148fab2e51b4f1cfc66da2a7c32981d1d3c083a803978268bb11fe4b86925e7a"
|
||||
"checksum memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "796fba70e76612589ed2ce7f45282f5af869e0fdd7cc6199fa1aa1f1d591ba9d"
|
||||
"checksum memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0f9dc261e2b62d7a622bf416ea3c5245cdd5d9a7fcc428c0d06804dfce1775b3"
|
||||
"checksum ndarray 0.11.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6c4b9c5e903f811c0df6bf5b414c6321d6b1fb31110d545a5c6003822594ec1a"
|
||||
"checksum nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "9a2228dca57108069a5262f2ed8bd2e82496d2e074a06d1ccc7ce1687b6ae0a2"
|
||||
"checksum num-complex 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "26ff8edeab9f1d8cf6b595e35138c2a389ea29f4f57a0e6bc44abf406e4b0077"
|
||||
"checksum num-integer 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)" = "f8d26da319fb45674985c78f1d1caf99aa4941f785d384a2ae36d0740bc3e2fe"
|
||||
"checksum num-iter 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)" = "4b226df12c5a59b63569dd57fafb926d91b385dfce33d8074a412411b689d593"
|
||||
|
|
@ -1227,6 +1315,8 @@ dependencies = [
|
|||
"checksum quote 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1eca14c727ad12702eb4b6bfb5a232287dcf8385cb8ca83a3eeaf6519c44c408"
|
||||
"checksum rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "eba5f8cb59cc50ed56be8880a5c7b496bfd9bd26394e176bc67884094145c2c5"
|
||||
"checksum rawpointer 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ebac11a9d2e11f2af219b8b8d833b76b1ea0e054aa0e8d8e9e4cbde353bdf019"
|
||||
"checksum rayon 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "485541959c8ecc49865526fe6c4de9653dd6e60d829d6edf0be228167b60372d"
|
||||
"checksum rayon-core 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9d24ad214285a7729b174ed6d3bcfcb80177807f959d95fafd5bfc5c4f201ac8"
|
||||
"checksum redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)" = "0d92eecebad22b767915e4d529f89f28ee96dbbf5a4810d2b844373f136417fd"
|
||||
"checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76"
|
||||
"checksum regex 0.1.80 (registry+https://github.com/rust-lang/crates.io-index)" = "4fd4ace6a8cf7860714a2c2280d6c1f7e6a413486c13298bbc86fd3da019402f"
|
||||
|
|
@ -1234,6 +1324,7 @@ dependencies = [
|
|||
"checksum rustc-demangle 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "11fb43a206a04116ffd7cfcf9bcb941f8eb6cc7ff667272246b0a1c74259a3cb"
|
||||
"checksum rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)" = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda"
|
||||
"checksum same-file 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "cfb6eded0b06a0b512c8ddbcf04089138c9b4362c2f696f3c3d76039d68f3637"
|
||||
"checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27"
|
||||
"checksum serde 0.9.15 (registry+https://github.com/rust-lang/crates.io-index)" = "34b623917345a631dc9608d5194cc206b3fe6c3554cd1c75b937e55e285254af"
|
||||
"checksum serde 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)" = "db99f3919e20faa51bb2996057f5031d8685019b5a06139b1ce761da671b8526"
|
||||
"checksum serde_codegen 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1f94de585a73dfc312ca77194209278a587bf90d3edc6c2d0fc479b0ed71d1f0"
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ structopt-derive = "0.2.4"
|
|||
serde = "1.0.27"
|
||||
serde_derive = "1.0.27"
|
||||
serde_json = "1.0.9"
|
||||
rayon = "1.0.0"
|
||||
dreammaker = { path = "../dreammaker" }
|
||||
dmm-tools = { path = "../tools" }
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
//! CLI tools, including a map renderer, using the same backend as the editor.
|
||||
|
||||
extern crate rayon;
|
||||
extern crate structopt;
|
||||
#[macro_use] extern crate structopt_derive;
|
||||
|
||||
|
|
@ -11,14 +12,14 @@ extern crate dreammaker as dm;
|
|||
#[macro_use] extern crate dmm_tools;
|
||||
|
||||
use std::fmt;
|
||||
use std::path::PathBuf;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::{AtomicIsize, Ordering};
|
||||
use std::path::Path;
|
||||
|
||||
use structopt::StructOpt;
|
||||
|
||||
use dm::objtree::ObjectTree;
|
||||
use dmm_tools::*;
|
||||
use dmm_tools::dmi::IconFile;
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// Main driver
|
||||
|
|
@ -33,15 +34,15 @@ fn main() {
|
|||
flame::dump_html(&mut std::io::BufWriter::new(std::fs::File::create(format!("{}/flame-graph.html", opt.output)).unwrap())).unwrap();
|
||||
}
|
||||
|
||||
std::process::exit(context.exit_status);
|
||||
std::process::exit(context.exit_status.into_inner() as i32);
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Context {
|
||||
dm_context: dm::Context,
|
||||
objtree: ObjectTree,
|
||||
icon_cache: HashMap<PathBuf, IconFile>,
|
||||
exit_status: i32,
|
||||
icon_cache: icon_cache::IconCache,
|
||||
exit_status: AtomicIsize,
|
||||
}
|
||||
|
||||
impl Context {
|
||||
|
|
@ -120,6 +121,10 @@ enum Command {
|
|||
#[structopt(long="disable", default_value="")]
|
||||
disable: String,
|
||||
|
||||
/// Set the number of jobs to run in parallel.
|
||||
#[structopt(long="jobs", default_value="1")]
|
||||
jobs: usize,
|
||||
|
||||
/// Run output through pngcrush automatically. Requires pngcrush.
|
||||
#[structopt(long="pngcrush")]
|
||||
pngcrush: bool,
|
||||
|
|
@ -215,16 +220,27 @@ fn run(opt: &Opt, command: &Command, context: &mut Context) {
|
|||
// --------------------------------------------------------------------
|
||||
Command::Minimap {
|
||||
ref output, min, max, ref enable, ref disable, ref files,
|
||||
pngcrush, optipng,
|
||||
jobs, pngcrush, optipng,
|
||||
} => {
|
||||
context.objtree(opt);
|
||||
let objtree = &context.objtree;
|
||||
let icon_cache = &context.icon_cache;
|
||||
let errors = &context.exit_status;
|
||||
|
||||
let render_passes = dmm_tools::render_passes::configure(enable, disable);
|
||||
for path in files.iter() {
|
||||
let path: &std::path::Path = path.as_ref();
|
||||
let render_passes = &dmm_tools::render_passes::configure(enable, disable);
|
||||
let paths: Vec<&Path> = files.iter().map(|p| p.as_ref()).collect();
|
||||
|
||||
let perform_job = move |path: &Path| {
|
||||
println!("{}", path.display());
|
||||
flame!(path.file_name().unwrap().to_string_lossy().into_owned());
|
||||
let mut map = dmm::Map::from_file(path).expect("DMM file missing");
|
||||
let map = match dmm::Map::from_file(path) {
|
||||
Ok(map) => map,
|
||||
Err(e) => {
|
||||
eprintln!("Failed to load {}:\n{}", path.display(), e);
|
||||
errors.fetch_add(1, Ordering::Relaxed);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let (dim_x, dim_y, dim_z) = map.dim_xyz();
|
||||
let mut min = min.unwrap_or(CoordArg { x: 0, y: 0, z: 0 });
|
||||
|
|
@ -240,15 +256,19 @@ fn run(opt: &Opt, command: &Command, context: &mut Context) {
|
|||
for z in (min.z - 1)..(max.z) {
|
||||
println!(" generating z={}", 1 + z);
|
||||
let minimap_context = minimap::Context {
|
||||
objtree: &context.objtree,
|
||||
objtree: &objtree,
|
||||
map: &map,
|
||||
grid: map.z_level(z),
|
||||
min: (min.x - 1, min.y - 1),
|
||||
max: (max.x - 1, max.y - 1),
|
||||
render_passes: &render_passes,
|
||||
};
|
||||
let image = minimap::generate(minimap_context, &mut context.icon_cache).unwrap();
|
||||
std::fs::create_dir_all(output).expect("Failed to create output directory");
|
||||
let image = minimap::generate(minimap_context, icon_cache).unwrap();
|
||||
if let Err(e) = std::fs::create_dir_all(output) {
|
||||
eprintln!("Failed to create output directory {}:\n{}", output, e);
|
||||
errors.fetch_add(1, Ordering::Relaxed);
|
||||
return;
|
||||
}
|
||||
let outfile = format!("{}/{}-{}.png", output, path.file_stem().unwrap().to_string_lossy(), 1 + z);
|
||||
println!(" saving {}", outfile);
|
||||
image.to_file(outfile.as_ref()).unwrap();
|
||||
|
|
@ -274,6 +294,23 @@ fn run(opt: &Opt, command: &Command, context: &mut Context) {
|
|||
.success(), "optipng failed");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if jobs > 1 {
|
||||
// Suboptimal due to mixing I/O in with what's meant for CPU
|
||||
// tasks, but it should get the job done for now.
|
||||
rayon::scope(|scope| {
|
||||
for path in paths {
|
||||
let perform_job = &perform_job;
|
||||
scope.spawn(move |_scope| {
|
||||
perform_job(path);
|
||||
});
|
||||
}
|
||||
});
|
||||
} else {
|
||||
for path in paths {
|
||||
perform_job(path);
|
||||
}
|
||||
}
|
||||
},
|
||||
// --------------------------------------------------------------------
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue