From 26dfa0120511bcff15d3de222baf382eb8fd49f7 Mon Sep 17 00:00:00 2001 From: Richard Feldman Date: Thu, 1 Oct 2020 22:14:04 -0400 Subject: [PATCH] Extract target triple and linking logic --- cli/src/build.rs | 116 +++++++++++++++++++++++++++ cli/src/lib.rs | 147 ++-------------------------------- cli/src/main.rs | 13 ++- compiler/build/src/lib.rs | 2 + compiler/build/src/link.rs | 66 +++++++++++++++ compiler/build/src/program.rs | 65 +-------------- compiler/build/src/target.rs | 76 ++++++++++++++++++ 7 files changed, 282 insertions(+), 203 deletions(-) create mode 100644 cli/src/build.rs create mode 100644 compiler/build/src/link.rs create mode 100644 compiler/build/src/target.rs diff --git a/cli/src/build.rs b/cli/src/build.rs new file mode 100644 index 0000000000..294122184b --- /dev/null +++ b/cli/src/build.rs @@ -0,0 +1,116 @@ +use bumpalo::Bump; +use roc_build::{link::link, program, target::target_triple_str}; +use roc_collections::all::MutMap; +use roc_gen::llvm::build::OptLevel; +use roc_load::file::LoadingProblem; +use std::fs; +use std::path::PathBuf; +use std::time::{Duration, SystemTime}; +use target_lexicon::Triple; + +fn report_timing(buf: &mut String, label: &str, duration: Duration) { + buf.push_str(&format!( + " {:.3} ms {}\n", + duration.as_secs_f64() * 1000.0, + label, + )); +} + +pub fn build_file( + target: &Triple, + src_dir: PathBuf, + filename: PathBuf, + opt_level: OptLevel, +) -> Result { + let compilation_start = SystemTime::now(); + let arena = Bump::new(); + + // Step 1: compile the app and generate the .o file + let subs_by_module = MutMap::default(); + + // Release builds use uniqueness optimizations + let stdlib = match opt_level { + OptLevel::Normal => roc_builtins::std::standard_stdlib(), + OptLevel::Optimize => roc_builtins::unique::uniq_stdlib(), + }; + let loaded = + roc_load::file::load(filename.clone(), &stdlib, src_dir.as_path(), subs_by_module)?; + let dest_filename = filename.with_file_name("roc_app.o"); + let buf = &mut String::with_capacity(1024); + + for (module_id, module_timing) in loaded.timings.iter() { + let module_name = loaded.interns.module_name(*module_id); + + buf.push_str(" "); + buf.push_str(module_name); + buf.push_str("\n"); + + report_timing(buf, "Read .roc file from disk", module_timing.read_roc_file); + report_timing(buf, "Parse header", module_timing.parse_header); + report_timing(buf, "Parse body", module_timing.parse_body); + report_timing(buf, "Canonicalize", module_timing.canonicalize); + report_timing(buf, "Constrain", module_timing.constrain); + report_timing(buf, "Solve", module_timing.solve); + report_timing(buf, "Other", module_timing.other()); + buf.push('\n'); + report_timing(buf, "Total", module_timing.total()); + } + + println!( + "\n\nCompilation finished! Here's how long each module took to compile:\n\n{}", + buf + ); + + program::gen( + &arena, + loaded, + filename, + Triple::host(), + &dest_filename, + opt_level, + ); + + let compilation_end = compilation_start.elapsed().unwrap(); + + println!( + "Finished compilation and code gen in {} ms\n", + compilation_end.as_millis() + ); + + let cwd = dest_filename.parent().unwrap(); + + // Step 2: link the precompiled host and compiled app + let host_input_path = cwd + .join("platform") + .join("host") + .join(target_triple_str(target)) + .join("host.o"); + let binary_path = cwd.join("app"); // TODO should be app.exe on Windows + + // TODO try to move as much of this linking as possible to the precompiled + // host, to minimize the amount of host-application linking required. + let cmd_result = // TODO use lld + link( + target, + binary_path.as_path(), + host_input_path.as_path(), + dest_filename.as_path(), + ) + .map_err(|_| { + todo!("gracefully handle `rustc` failing to spawn."); + })? + .wait() + .map_err(|_| { + todo!("gracefully handle error after `rustc` spawned"); + }); + + // Clean up the leftover .o file from the Roc, if possible. + // (If cleaning it up fails, that's fine. No need to take action.) + // TODO compile the dest_filename to a tmpdir, as an extra precaution. + let _ = fs::remove_file(dest_filename); + + // If the cmd errored out, return the Err. + cmd_result?; + + Ok(binary_path) +} diff --git a/cli/src/lib.rs b/cli/src/lib.rs index 6e227afad6..e821a13017 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -1,21 +1,16 @@ #[macro_use] extern crate clap; -use bumpalo::Bump; use clap::ArgMatches; use clap::{App, Arg}; -use roc_build::program::gen; -use roc_collections::all::MutMap; use roc_gen::llvm::build::OptLevel; -use roc_load::file::LoadingProblem; -use std::fs; -use std::io::{self, ErrorKind}; -use std::path::{Path, PathBuf}; +use std::io; +use std::path::Path; use std::process; use std::process::Command; -use std::time::{Duration, SystemTime}; use target_lexicon::Triple; +pub mod build; pub mod repl; pub static FLAG_OPTIMIZE: &str = "optimize"; @@ -67,7 +62,7 @@ pub fn build_app<'a>() -> App<'a> { ) } -pub fn build(matches: &ArgMatches, run_after_build: bool) -> io::Result<()> { +pub fn build(target: &Triple, matches: &ArgMatches, run_after_build: bool) -> io::Result<()> { let filename = matches.value_of(FLAG_ROC_FILE).unwrap(); let opt_level = if matches.is_present(FLAG_OPTIMIZE) { OptLevel::Optimize @@ -79,7 +74,7 @@ pub fn build(matches: &ArgMatches, run_after_build: bool) -> io::Result<()> { // Spawn the root task let path = path.canonicalize().unwrap_or_else(|err| { - use ErrorKind::*; + use io::ErrorKind::*; match err.kind() { NotFound => { @@ -96,8 +91,8 @@ pub fn build(matches: &ArgMatches, run_after_build: bool) -> io::Result<()> { } }); - let binary_path = - build_file(src_dir, path, opt_level).expect("TODO gracefully handle build_file failing"); + let binary_path = build::build_file(target, src_dir, path, opt_level) + .expect("TODO gracefully handle build_file failing"); if run_after_build { // Run the compiled app @@ -110,131 +105,3 @@ pub fn build(matches: &ArgMatches, run_after_build: bool) -> io::Result<()> { Ok(()) } - -fn report_timing(buf: &mut String, label: &str, duration: Duration) { - buf.push_str(&format!( - " {:.3} ms {}\n", - duration.as_secs_f64() * 1000.0, - label, - )); -} - -fn build_file( - src_dir: PathBuf, - filename: PathBuf, - opt_level: OptLevel, -) -> Result { - let compilation_start = SystemTime::now(); - let arena = Bump::new(); - - // Step 1: compile the app and generate the .o file - let subs_by_module = MutMap::default(); - - // Release builds use uniqueness optimizations - let stdlib = match opt_level { - OptLevel::Normal => roc_builtins::std::standard_stdlib(), - OptLevel::Optimize => roc_builtins::unique::uniq_stdlib(), - }; - let loaded = - roc_load::file::load(filename.clone(), &stdlib, src_dir.as_path(), subs_by_module)?; - let dest_filename = filename.with_file_name("roc_app.o"); - let buf = &mut String::with_capacity(1024); - - for (module_id, module_timing) in loaded.timings.iter() { - let module_name = loaded.interns.module_name(*module_id); - - buf.push_str(" "); - buf.push_str(module_name); - buf.push_str("\n"); - - report_timing(buf, "Read .roc file from disk", module_timing.read_roc_file); - report_timing(buf, "Parse header", module_timing.parse_header); - report_timing(buf, "Parse body", module_timing.parse_body); - report_timing(buf, "Canonicalize", module_timing.canonicalize); - report_timing(buf, "Constrain", module_timing.constrain); - report_timing(buf, "Solve", module_timing.solve); - report_timing(buf, "Other", module_timing.other()); - buf.push('\n'); - report_timing(buf, "Total", module_timing.total()); - } - - println!( - "\n\nCompilation finished! Here's how long each module took to compile:\n\n{}", - buf - ); - - gen( - &arena, - loaded, - filename, - Triple::host(), - &dest_filename, - opt_level, - ); - - let compilation_end = compilation_start.elapsed().unwrap(); - - println!( - "Finished compilation and code gen in {} ms\n", - compilation_end.as_millis() - ); - - let cwd = dest_filename.parent().unwrap(); - - // Step 2: link the precompiled host and compiled app - let arch = "x86_64"; // TODO determine this based on target - let target_triple = "x86_64-unknown-linux-gnu"; - let host_input_path = cwd - .join("platform") - .join("host") - .join(target_triple) - .join("host.o"); - let binary_path = cwd.join("app"); // TODO should be app.exe on Windows - - // TODO try to move as much of this linking as possible to the precompiled - // host, to minimize the amount of host-application linking required. - let cmd_result = Command::new("ld") // TODO use lld - .args(&[ - "-arch", - arch, - "/usr/lib/x86_64-linux-gnu/crti.o", - "/usr/lib/x86_64-linux-gnu/crtn.o", - "/usr/lib/x86_64-linux-gnu/Scrt1.o", - "-dynamic-linker", - "/lib64/ld-linux-x86-64.so.2", - // Libraries - see https://github.com/rtfeldman/roc/pull/554#discussion_r496365925 - // for discussion and further references - "-lc", - "-lm", - "-lpthread", - "-ldl", - "-lrt", - "-lutil", - "-lc_nonshared", - // "-lc++", // TODO shouldn't we need this? - // "-lgcc", // TODO will eventually need compiler_rt from gcc or something - see https://github.com/rtfeldman/roc/pull/554#discussion_r496370840 - // "-lunwind", // TODO will eventually need this, see https://github.com/rtfeldman/roc/pull/554#discussion_r496370840 - "-o", - binary_path.as_path().to_str().unwrap(), // app - host_input_path.as_path().to_str().unwrap(), // host.o - dest_filename.as_path().to_str().unwrap(), // roc_app.o - ]) - .spawn() - .map_err(|_| { - todo!("gracefully handle `rustc` failing to spawn."); - })? - .wait() - .map_err(|_| { - todo!("gracefully handle error after `rustc` spawned"); - }); - - // Clean up the leftover .o file from the Roc, if possible. - // (If cleaning it up fails, that's fine. No need to take action.) - // TODO compile the dest_filename to a tmpdir, as an extra precaution. - let _ = fs::remove_file(dest_filename); - - // If the cmd errored out, return the Err. - cmd_result?; - - Ok(binary_path) -} diff --git a/cli/src/main.rs b/cli/src/main.rs index dd92325b72..3a07efd95a 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -1,14 +1,23 @@ use roc_cli::{build, build_app, repl, DIRECTORY_OR_FILES}; use std::io; use std::path::Path; +use target_lexicon::Triple; fn main() -> io::Result<()> { let matches = build_app().get_matches(); match matches.subcommand_name() { None => roc_editor::launch(&[]), - Some("build") => build(matches.subcommand_matches("build").unwrap(), false), - Some("run") => build(matches.subcommand_matches("run").unwrap(), true), + Some("build") => build( + &Triple::host(), + matches.subcommand_matches("build").unwrap(), + false, + ), + Some("run") => build( + &Triple::host(), + matches.subcommand_matches("run").unwrap(), + true, + ), Some("repl") => repl::main(), Some("edit") => { match matches diff --git a/compiler/build/src/lib.rs b/compiler/build/src/lib.rs index 15ced922ef..896bcc6c21 100644 --- a/compiler/build/src/lib.rs +++ b/compiler/build/src/lib.rs @@ -10,4 +10,6 @@ // and encouraging shortcuts here creates bad incentives. I would rather temporarily // re-enable this when working on performance optimizations than have it block PRs. #![allow(clippy::large_enum_variant)] +pub mod link; pub mod program; +pub mod target; diff --git a/compiler/build/src/link.rs b/compiler/build/src/link.rs new file mode 100644 index 0000000000..705ffdee19 --- /dev/null +++ b/compiler/build/src/link.rs @@ -0,0 +1,66 @@ +use crate::target::arch_str; +use std::io; +use std::path::Path; +use std::process::{Child, Command}; +use target_lexicon::{Architecture, OperatingSystem, Triple}; + +pub fn link( + target: &Triple, + binary_path: &Path, + host_input_path: &Path, + dest_filename: &Path, +) -> io::Result { + match target { + Triple { + architecture: Architecture::X86_64, + operating_system: OperatingSystem::Linux, + .. + } => link_linux( + arch_str(target), + binary_path, + host_input_path, + dest_filename, + ), + Triple { + architecture: Architecture::X86_64, + operating_system: OperatingSystem::Darwin, + .. + } => todo!("link macos"), + _ => panic!("TODO gracefully handle unsupported target: {:?}", target), + } +} + +fn link_linux( + arch: &str, + binary_path: &Path, + host_input_path: &Path, + dest_filename: &Path, +) -> io::Result { + Command::new("ld") + .args(&[ + "-arch", + arch, + "/usr/lib/x86_64-linux-gnu/crti.o", + "/usr/lib/x86_64-linux-gnu/crtn.o", + "/usr/lib/x86_64-linux-gnu/Scrt1.o", + "-dynamic-linker", + "/lib64/ld-linux-x86-64.so.2", + // Libraries - see https://github.com/rtfeldman/roc/pull/554#discussion_r496365925 + // for discussion and further references + "-lc", + "-lm", + "-lpthread", + "-ldl", + "-lrt", + "-lutil", + "-lc_nonshared", + // "-lc++", // TODO shouldn't we need this? + // "-lgcc", // TODO will eventually need compiler_rt from gcc or something - see https://github.com/rtfeldman/roc/pull/554#discussion_r496370840 + // "-lunwind", // TODO will eventually need this, see https://github.com/rtfeldman/roc/pull/554#discussion_r496370840 + "-o", + binary_path.to_str().unwrap(), // app + host_input_path.to_str().unwrap(), // host.o + dest_filename.to_str().unwrap(), // roc_app.o + ]) + .spawn() +} diff --git a/compiler/build/src/program.rs b/compiler/build/src/program.rs index 75df8a8343..c08e1df1f3 100644 --- a/compiler/build/src/program.rs +++ b/compiler/build/src/program.rs @@ -1,8 +1,7 @@ +use crate::target; use bumpalo::Bump; use inkwell::context::Context; -use inkwell::targets::{ - CodeModel, FileType, InitializationConfig, RelocMode, Target, TargetTriple, -}; +use inkwell::targets::{CodeModel, FileType, RelocMode}; use inkwell::OptimizationLevel; use roc_collections::all::default_hasher; use roc_gen::layout_id::LayoutIds; @@ -12,7 +11,7 @@ use roc_mono::ir::{Env, PartialProc, Procs}; use roc_mono::layout::{Layout, LayoutCache}; use std::collections::HashSet; use std::path::{Path, PathBuf}; -use target_lexicon::{Architecture, OperatingSystem, Triple}; +use target_lexicon::Triple; // TODO how should imported modules factor into this? What if those use builtins too? // TODO this should probably use more helper functions @@ -295,66 +294,10 @@ pub fn gen( // Emit the .o file - // NOTE: arch_str is *not* the same as the beginning of the magic target triple - // string! For example, if it's "x86-64" here, the magic target triple string - // will begin with "x86_64" (with an underscore) instead. - let arch_str = match target.architecture { - Architecture::X86_64 => { - Target::initialize_x86(&InitializationConfig::default()); - - "x86-64" - } - Architecture::Arm(_) if cfg!(feature = "target-arm") => { - // NOTE: why not enable arm and wasm by default? - // - // We had some trouble getting them to link properly. This may be resolved in the - // future, or maybe it was just some weird configuration on one machine. - Target::initialize_arm(&InitializationConfig::default()); - - "arm" - } - Architecture::Wasm32 if cfg!(feature = "target-webassembly") => { - Target::initialize_webassembly(&InitializationConfig::default()); - - "wasm32" - } - _ => panic!( - "TODO gracefully handle unsupported target architecture: {:?}", - target.architecture - ), - }; - let opt = OptimizationLevel::Aggressive; let reloc = RelocMode::Default; let model = CodeModel::Default; - - // Best guide I've found on how to determine these magic strings: - // - // https://stackoverflow.com/questions/15036909/clang-how-to-list-supported-target-architectures - let target_triple_str = match target { - Triple { - architecture: Architecture::X86_64, - operating_system: OperatingSystem::Linux, - .. - } => "x86_64-unknown-linux-gnu", - Triple { - architecture: Architecture::X86_64, - operating_system: OperatingSystem::Darwin, - .. - } => "x86_64-unknown-darwin10", - _ => panic!("TODO gracefully handle unsupported target: {:?}", target), - }; - let target_machine = Target::from_name(arch_str) - .unwrap() - .create_target_machine( - &TargetTriple::create(target_triple_str), - arch_str, - "+avx2", // TODO this string was used uncritically from an example, and should be reexamined - opt, - reloc, - model, - ) - .unwrap(); + let target_machine = target::target_machine(&target, opt, reloc, model).unwrap(); target_machine .write_to_file(&env.module, FileType::Object, &dest_filename) diff --git a/compiler/build/src/target.rs b/compiler/build/src/target.rs new file mode 100644 index 0000000000..48aad60072 --- /dev/null +++ b/compiler/build/src/target.rs @@ -0,0 +1,76 @@ +use inkwell::targets::{ + CodeModel, InitializationConfig, RelocMode, Target, TargetMachine, TargetTriple, +}; +use inkwell::OptimizationLevel; +use target_lexicon::{Architecture, OperatingSystem, Triple}; + +pub fn target_triple_str(target: &Triple) -> &'static str { + // Best guide I've found on how to determine these magic strings: + // + // https://stackoverflow.com/questions/15036909/clang-how-to-list-supported-target-architectures + match target { + Triple { + architecture: Architecture::X86_64, + operating_system: OperatingSystem::Linux, + .. + } => "x86_64-unknown-linux-gnu", + Triple { + architecture: Architecture::X86_64, + operating_system: OperatingSystem::Darwin, + .. + } => "x86_64-unknown-darwin10", + _ => panic!("TODO gracefully handle unsupported target: {:?}", target), + } +} + +/// NOTE: arch_str is *not* the same as the beginning of the magic target triple +/// string! For example, if it's "x86-64" here, the magic target triple string +/// will begin with "x86_64" (with an underscore) instead. +pub fn arch_str(target: &Triple) -> &'static str { + // Best guide I've found on how to determine these magic strings: + // + // https://stackoverflow.com/questions/15036909/clang-how-to-list-supported-target-architectures + match target.architecture { + Architecture::X86_64 => { + Target::initialize_x86(&InitializationConfig::default()); + + "x86-64" + } + Architecture::Arm(_) if cfg!(feature = "target-arm") => { + // NOTE: why not enable arm and wasm by default? + // + // We had some trouble getting them to link properly. This may be resolved in the + // future, or maybe it was just some weird configuration on one machine. + Target::initialize_arm(&InitializationConfig::default()); + + "arm" + } + Architecture::Wasm32 if cfg!(feature = "target-webassembly") => { + Target::initialize_webassembly(&InitializationConfig::default()); + + "wasm32" + } + _ => panic!( + "TODO gracefully handle unsupported target architecture: {:?}", + target.architecture + ), + } +} + +pub fn target_machine( + target: &Triple, + opt: OptimizationLevel, + reloc: RelocMode, + model: CodeModel, +) -> Option { + let arch = arch_str(target); + + Target::from_name(arch).unwrap().create_target_machine( + &TargetTriple::create(target_triple_str(target)), + arch, + "+avx2", // TODO this string was used uncritically from an example, and should be reexamined + opt, + reloc, + model, + ) +}