language-servers/roc
1
0
Fork 0
mirror of https://github.com/roc-lang/roc.git synced 2025-08-03 19:58:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

Merge pull request #4565 from roc-lang/more-standalone

Browse source 
Use include_bytes! so builtin hosts live in binary
This commit is contained in:
Folkert de Vries 2022-11-23 13:01:39 +01:00 • committed by GitHub
commit 099ab4938f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 146 additions and 72 deletions
Download patch file Download diff file Expand all files Collapse all files

1
Cargo.lock generated
View file

@ -3993,6 +3993,7 @@ dependencies = [
"roc_target",
"roc_types",
"roc_utils",
"tempfile",
"wasi_libc_sys",
"wasm-bindgen",
"wasm-bindgen-futures",

26
crates/cli/src/build.rs
View file

@ -17,7 +17,6 @@ use roc_target::TargetInfo;
use std::time::{Duration, Instant};
use std::{path::PathBuf, thread::JoinHandle};
use target_lexicon::Triple;
use tempfile::Builder;
fn report_timing(buf: &mut String, label: &str, duration: Duration) {
use std::fmt::Write;
@ -331,7 +330,7 @@ pub fn build_file<'a>(
problems
}
(LinkingStrategy::Legacy, _) => {
let app_o_file = Builder::new()
let app_o_file = tempfile::Builder::new()
.prefix("roc_app")
.suffix(&format!(".{}", app_extension))
.tempfile()
@ -345,20 +344,35 @@ pub fn build_file<'a>(
app_o_file.to_str().unwrap(),
];
let str_host_obj_path = bitcode::get_builtins_host_obj_path();
let builtins_host_tempfile = {
#[cfg(unix)]
{
bitcode::host_unix_tempfile()
}
#[cfg(windows)]
{
bitcode::host_windows_tempfile()
}
}
.expect("failed to write host builtins object to tempfile");
if matches!(code_gen_options.backend, program::CodeGenBackend::Assembly) {
inputs.push(&str_host_obj_path);
inputs.push(builtins_host_tempfile.path().to_str().unwrap());
}
let (mut child, _) = // TODO use lld
link(target, binary_path.clone(), &inputs, link_type)
.map_err(|_| todo!("gracefully handle `ld` failing to spawn."))?;
link(target, binary_path.clone(), &inputs, link_type)
.map_err(|_| todo!("gracefully handle `ld` failing to spawn."))?;
let exit_status = child
.wait()
.map_err(|_| todo!("gracefully handle error after `ld` spawned"))?;
// Extend the lifetime of the tempfile so it doesn't get dropped
// (and thus deleted) before the child process is done using it!
let _ = builtins_host_tempfile;
if exit_status.success() {
problems
} else {

57
crates/compiler/build/src/link.rs
View file

@ -107,6 +107,7 @@ pub fn build_zig_host_native(
target: &str,
opt_level: OptLevel,
shared_lib_path: Option<&Path>,
builtins_host_path: &Path,
) -> Command {
let mut zig_cmd = zig();
zig_cmd
@ -118,18 +119,12 @@ pub fn build_zig_host_native(
// with LLVM, the builtins are already part of the roc app,
// but with the dev backend, they are missing. To minimize work,
// we link them as part of the host executable
let builtins_obj = if target.contains("windows") {
bitcode::get_builtins_windows_obj_path()
} else {
bitcode::get_builtins_host_obj_path()
};
zig_cmd.args([
"build-exe",
"-fPIE",
"-rdynamic", // make sure roc_alloc and friends are exposed
shared_lib_path.to_str().unwrap(),
&builtins_obj,
builtins_host_path.to_str().unwrap(),
]);
} else {
zig_cmd.args(["build-obj", "-fPIC"]);
@ -185,6 +180,7 @@ pub fn build_zig_host_native(
target: &str,
opt_level: OptLevel,
shared_lib_path: Option<&Path>,
builtins_host_path: &Path,
) -> Command {
// to prevent `clang failed with stderr: zig: error: unable to make temporary file: No such file or directory`
let env_userprofile = env::var("USERPROFILE").unwrap_or_else(|_| "".to_string());
@ -201,7 +197,7 @@ pub fn build_zig_host_native(
"build-exe",
// "-fPIE", PIE seems to fail on windows
shared_lib_path.to_str().unwrap(),
&bitcode::get_builtins_windows_obj_path(),
builtins_host_path.to_str().unwrap(),
]);
} else {
zig_cmd.args(&["build-obj"]);
@ -244,6 +240,7 @@ pub fn build_zig_host_native(
_target: &str,
opt_level: OptLevel,
shared_lib_path: Option<&Path>,
builtins_host_path: &Path,
// For compatibility with the non-macOS def above. Keep these in sync.
) -> Command {
use serde_json::Value;
@ -296,7 +293,7 @@ pub fn build_zig_host_native(
"build-exe",
"-fPIE",
shared_lib_path.to_str().unwrap(),
&bitcode::get_builtins_host_obj_path(),
builtins_host_path.to_str().unwrap(),
]);
} else {
zig_cmd.args(&["build-obj"]);
@ -400,6 +397,7 @@ pub fn build_c_host_native(
sources: &[&str],
opt_level: OptLevel,
shared_lib_path: Option<&Path>,
builtins_host_path: &Path,
) -> Command {
let mut clang_cmd = clang();
clang_cmd
@ -426,6 +424,7 @@ pub fn build_c_host_native(
get_target_str(target),
opt_level,
Some(shared_lib_path),
builtins_host_path,
);
}
_ => {
@ -436,7 +435,7 @@ pub fn build_c_host_native(
// linking the built-ins led to a surgical linker bug for
// optimized builds. Disabling until it is needed for dev
// builds.
// &bitcode::get_builtins_host_obj_path(),
// builtins_host_path,
"-fPIE",
"-pie",
"-lm",
@ -554,6 +553,19 @@ pub fn rebuild_host(
let env_home = env::var("HOME").unwrap_or_else(|_| "".to_string());
let env_cpath = env::var("CPATH").unwrap_or_else(|_| "".to_string());
let builtins_host_tempfile = {
#[cfg(windows)]
{
bitcode::host_windows_tempfile()
}
#[cfg(unix)]
{
bitcode::host_unix_tempfile()
}
}
.expect("failed to write host builtins object to tempfile");
if zig_host_src.exists() {
// Compile host.zig
@ -591,6 +603,7 @@ pub fn rebuild_host(
get_target_str(target),
opt_level,
shared_lib_path,
builtins_host_tempfile.path(),
),
Architecture::X86_32(_) => build_zig_host_native(
&env_path,
@ -601,8 +614,8 @@ pub fn rebuild_host(
"i386-linux-musl",
opt_level,
shared_lib_path,
builtins_host_tempfile.path(),
),
Architecture::Aarch64(_) => build_zig_host_native(
&env_path,
&env_home,
@ -612,11 +625,12 @@ pub fn rebuild_host(
target_zig_str(target),
opt_level,
shared_lib_path,
builtins_host_tempfile.path(),
),
_ => internal_error!("Unsupported architecture {:?}", target.architecture),
};
run_build_command(zig_cmd, "host.zig", 0)
run_build_command(zig_cmd, "host.zig", 0);
} else if cargo_host_src.exists() {
// Compile and link Cargo.toml, if it exists
let cargo_dir = host_input_path.parent().unwrap();
@ -679,6 +693,7 @@ pub fn rebuild_host(
&[c_host_src.to_str().unwrap()],
opt_level,
shared_lib_path,
builtins_host_tempfile.path(),
);
run_build_command(clang_cmd, "host.c", 0);
@ -733,6 +748,7 @@ pub fn rebuild_host(
],
opt_level,
shared_lib_path,
builtins_host_tempfile.path(),
);
run_build_command(clang_cmd, "host.c", 0);
} else {
@ -745,6 +761,7 @@ pub fn rebuild_host(
&[c_host_src.to_str().unwrap()],
opt_level,
shared_lib_path,
builtins_host_tempfile.path(),
);
run_build_command(clang_cmd, "host.c", 0);
@ -780,6 +797,7 @@ pub fn rebuild_host(
&[c_host_src.to_str().unwrap()],
opt_level,
shared_lib_path,
builtins_host_tempfile.path(),
);
run_build_command(clang_cmd, "host.c", 0);
@ -801,6 +819,10 @@ pub fn rebuild_host(
run_build_command(swiftc_cmd, "host.swift", 0);
}
// Extend the lifetime of the tempfile so it doesn't get dropped
// (and thus deleted) before the build process is done using it!
let _ = builtins_host_tempfile;
host_dest
}
@ -1353,10 +1375,13 @@ pub fn preprocess_host_wasm32(host_input_path: &Path, preprocessed_host_path: &P
(but seems to be an unofficial API)
*/
let builtins_host_tempfile =
bitcode::host_wasm_tempfile().expect("failed to write host builtins object to tempfile");
let mut zig_cmd = zig();
let args = &[
"wasm-ld",
&bitcode::get_builtins_wasm32_obj_path(),
builtins_host_tempfile.path().to_str().unwrap(),
host_input,
WASI_LIBC_PATH,
WASI_COMPILER_RT_PATH, // builtins need __multi3, __udivti3, __fixdfti
@ -1373,7 +1398,11 @@ pub fn preprocess_host_wasm32(host_input_path: &Path, preprocessed_host_path: &P
// println!("\npreprocess_host_wasm32");
// println!("zig {}\n", args.join(" "));
run_build_command(zig_cmd, output_file, 0)
run_build_command(zig_cmd, output_file, 0);
// Extend the lifetime of the tempfile so it doesn't get dropped
// (and thus deleted) before the Zig process is done using it!
let _ = builtins_host_tempfile;
}
fn run_build_command(mut command: Command, file_to_build: &str, flaky_fail_counter: usize) {

3
crates/compiler/builtins/Cargo.toml
View file

@ -12,6 +12,7 @@ roc_region = { path = "../region" }
roc_module = { path = "../module" }
roc_target = { path = "../roc_target" }
roc_utils = { path = "../../utils" }
tempfile.workspace = true
[build-dependencies]
# dunce can be removed once ziglang/zig#5109 is fixed
@ -19,4 +20,4 @@ dunce = "1.0.3"
roc_utils = { path = "../../utils" }
[target.'cfg(target_os = "macos")'.build-dependencies]
tempfile = "3.2.0"
tempfile.workspace = true

3
crates/compiler/builtins/bitcode/.gitignore vendored
View file

@ -1,7 +1,4 @@
zig-cache
src/zig-cache
benchmark/zig-cache
builtins.ll
builtins.bc
builtins.o
dec

19
crates/compiler/builtins/build.rs
View file

@ -134,24 +134,17 @@ fn generate_bc_file(bitcode_path: &Path, zig_object: &str, file_name: &str) {
pub fn get_lib_dir() -> PathBuf {
// Currently we have the OUT_DIR variable which points to `/target/debug/build/roc_builtins-*/out/`.
// So we just need to shed a 3 of the outer layers to get `/target/debug/` and then add `lib`.
let out_dir = env::var_os("OUT_DIR").unwrap();
// So we just need to add "/bitcode" to that.
let dir = PathBuf::from(env::var_os("OUT_DIR").unwrap()).join("bitcode");
let lib_path = Path::new(&out_dir)
.parent()
.and_then(|path| path.parent())
.and_then(|path| path.parent())
.unwrap()
.join("lib");
// create dir if it does not exist
fs::create_dir_all(&dir).expect("Failed to make $OUT_DIR/bitcode dir.");
// create dir of it does not exist
fs::create_dir_all(lib_path.clone()).expect("Failed to make lib dir.");
lib_path
dir
}
fn copy_zig_builtins_to_target_dir(bitcode_path: &Path) {
// To enable roc to find the zig biultins, we want them to be moved to a folder next to the roc executable.
// To enable roc to find the zig builtins, we want them to be moved to a folder next to the roc executable.
// So if <roc_folder>/roc is the executable. The zig files will be in <roc_folder>/lib/*.zig
let target_profile_dir = get_lib_dir();

64
crates/compiler/builtins/src/bitcode.rs
View file

@ -1,39 +1,55 @@
use roc_module::symbol::Symbol;
use roc_target::TargetInfo;
use roc_utils::get_lib_path;
use std::ops::Index;
use tempfile::NamedTempFile;
const LIB_DIR_ERROR: &str = "Failed to find the lib directory. Did you copy the roc binary without also copying the lib directory?\nIf you built roc from source, the lib dir should be in target/release.\nIf not, the lib dir should be included in the release tar.gz file.";
pub const HOST_WASM: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/bitcode/builtins-wasm32.o"));
// TODO: in the future, we should use Zig's cross-compilation to generate and store these
// for all targets, so that we can do cross-compilation!
#[cfg(unix)]
pub const HOST_UNIX: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/bitcode/builtins-host.o"));
#[cfg(windows)]
pub const HOST_WINDOWS: &[u8] = include_bytes!(concat!(
env!("OUT_DIR"),
"/bitcode/builtins-windows-x86_64.obj"
));
pub fn get_builtins_host_obj_path() -> String {
let builtins_host_path = get_lib_path().expect(LIB_DIR_ERROR).join("builtins-host.o");
pub fn host_wasm_tempfile() -> std::io::Result<NamedTempFile> {
let tempfile = tempfile::Builder::new()
.prefix("host_bitcode")
.suffix(".wasm")
.rand_bytes(8)
.tempfile()?;
builtins_host_path
.into_os_string()
.into_string()
.expect("Failed to convert builtins_host_path to str")
std::fs::write(tempfile.path(), HOST_WASM)?;
Ok(tempfile)
}
pub fn get_builtins_windows_obj_path() -> String {
let builtins_host_path = get_lib_path()
.expect(LIB_DIR_ERROR)
.join("builtins-windows-x86_64.obj");
#[cfg(unix)]
pub fn host_unix_tempfile() -> std::io::Result<NamedTempFile> {
let tempfile = tempfile::Builder::new()
.prefix("host_bitcode")
.suffix(".o")
.rand_bytes(8)
.tempfile()?;
builtins_host_path
.into_os_string()
.into_string()
.expect("Failed to convert builtins_host_path to str")
std::fs::write(tempfile.path(), HOST_UNIX)?;
Ok(tempfile)
}
pub fn get_builtins_wasm32_obj_path() -> String {
let builtins_wasm32_path = get_lib_path()
.expect(LIB_DIR_ERROR)
.join("builtins-wasm32.o");
#[cfg(windows)]
pub fn host_windows_tempfile() -> std::io::Result<NamedTempFile> {
let tempfile = tempfile::Builder::new()
.prefix("host_bitcode")
.suffix(".obj")
.rand_bytes(8)
.tempfile()?;
builtins_wasm32_path
.into_os_string()
.into_string()
.expect("Failed to convert builtins_wasm32_path to str")
std::fs::write(tempfile.path(), HOST_WINDOWS)?;
Ok(tempfile)
}
#[derive(Debug, Default, Copy, Clone)]

1
crates/compiler/test_gen/Cargo.toml
View file

@ -14,6 +14,7 @@ path = "src/tests.rs"
roc_builtins = { path = "../builtins" }
roc_utils = { path = "../../utils" }
wasi_libc_sys = { path = "../../wasi-libc-sys" }
tempfile.workspace = true
[dev-dependencies]
roc_gen_llvm = { path = "../gen_llvm" }

9
crates/compiler/test_gen/build.rs
View file

@ -100,9 +100,12 @@ fn build_wasm_test_host() {
let mut outfile = PathBuf::from(&out_dir).join(PLATFORM_FILENAME);
outfile.set_extension("wasm");
let builtins_host_tempfile =
bitcode::host_wasm_tempfile().expect("failed to write host builtins object to tempfile");
run_zig(&[
"wasm-ld",
&bitcode::get_builtins_wasm32_obj_path(),
builtins_host_tempfile.path().to_str().unwrap(),
platform_path.to_str().unwrap(),
WASI_COMPILER_RT_PATH,
WASI_LIBC_PATH,
@ -111,6 +114,10 @@ fn build_wasm_test_host() {
"--no-entry",
"--relocatable",
]);
// Extend the lifetime of the tempfile so it doesn't get dropped
// (and thus deleted) before the Zig process is done using it!
let _ = builtins_host_tempfile;
}
fn build_wasm_platform(out_dir: &str, source_path: &str) -> PathBuf {

9
crates/compiler/test_gen/src/helpers/dev.rs
View file

@ -193,7 +193,8 @@ pub fn helper(
.expect("failed to build output object");
std::fs::write(&app_o_file, module_out).expect("failed to write object to file");
// std::fs::copy(&app_o_file, "/tmp/app.o").unwrap();
let builtins_host_tempfile =
bitcode::host_unix_tempfile().expect("failed to write host builtins object to tempfile");
let (mut child, dylib_path) = link(
&target,
@ -202,7 +203,7 @@ pub fn helper(
// With the current method all methods are kept and it adds about 100k to all outputs.
&[
app_o_file.to_str().unwrap(),
&bitcode::get_builtins_host_obj_path(),
builtins_host_tempfile.path().to_str().unwrap(),
],
LinkType::Dylib,
)
@ -210,6 +211,10 @@ pub fn helper(
child.wait().unwrap();
// Extend the lifetime of the tempfile so it doesn't get dropped
// (and thus deleted) before the linking process is done using it!
let _ = builtins_host_tempfile;
// Load the dylib
let path = dylib_path.as_path().to_str().unwrap();

12
crates/linker/src/generate_dylib/macho.rs
View file

@ -4,19 +4,17 @@ use roc_error_macros::internal_error;
use std::path::Path;
use std::process::Command;
use target_lexicon::Triple;
use tempfile::Builder;
// TODO: Eventually do this from scratch and in memory instead of with ld.
pub fn create_dylib_macho(
custom_names: &[String],
triple: &Triple,
) -> object::read::Result<Vec<u8>> {
let dummy_obj_file = Builder::new()
let dummy_obj_file = tempfile::Builder::new()
.prefix("roc_lib")
.suffix(".o")
.tempfile()
.unwrap_or_else(|e| internal_error!("{}", e));
let dummy_obj_file = dummy_obj_file.path();
let tmp = tempfile::tempdir().unwrap_or_else(|e| internal_error!("{}", e));
let dummy_lib_file = tmp.path().to_path_buf().with_file_name("libapp.so");
@ -47,7 +45,7 @@ pub fn create_dylib_macho(
}
std::fs::write(
dummy_obj_file,
dummy_obj_file.path(),
out_object.write().expect("failed to build output object"),
)
.expect("failed to write object to file");
@ -70,13 +68,17 @@ pub fn create_dylib_macho(
.args([
ld_flag_soname,
dummy_lib_file.file_name().unwrap().to_str().unwrap(),
dummy_obj_file.to_str().unwrap(),
dummy_obj_file.path().to_str().unwrap(),
"-o",
dummy_lib_file.to_str().unwrap(),
])
.output()
.unwrap();
// Extend the lifetime of the tempfile so it doesn't get dropped
// (and thus deleted) before the linker process is done using it!
let _ = dummy_obj_file;
if !output.status.success() {
match std::str::from_utf8(&output.stderr) {
Ok(stderr) => panic!(

3
crates/repl_wasm/Cargo.toml
View file

@ -13,9 +13,10 @@ crate-type = ["cdylib"]
roc_builtins = {path = "../compiler/builtins"}
roc_utils = {path = "../utils"}
wasi_libc_sys = { path = "../wasi-libc-sys" }
tempfile.workspace = true
[dependencies]
bumpalo.workspace = true
bumpalo.workspace = true
console_error_panic_hook = {version = "0.1.7", optional = true}
futures = {version = "0.3.24", optional = true}
js-sys = "0.3.60"

9
crates/repl_wasm/build.rs
View file

@ -23,10 +23,13 @@ fn main() {
pre_linked_binary_path.extend(["pre_linked_binary"]);
pre_linked_binary_path.set_extension("o");
let builtins_host_tempfile =
bitcode::host_wasm_tempfile().expect("failed to write host builtins object to tempfile");
let output = Command::new(&zig_executable())
.args([
"wasm-ld",
&bitcode::get_builtins_wasm32_obj_path(),
builtins_host_tempfile.path().to_str().unwrap(),
platform_obj.to_str().unwrap(),
WASI_COMPILER_RT_PATH,
WASI_LIBC_PATH,
@ -39,6 +42,10 @@ fn main() {
.output()
.unwrap();
// Extend the lifetime of the tempfile so it doesn't get dropped
// (and thus deleted) before the Zig process is done using it!
let _ = builtins_host_tempfile;
assert!(output.status.success(), "{:#?}", output);
assert!(output.stdout.is_empty(), "{:#?}", output);
assert!(output.stderr.is_empty(), "{:#?}", output);

2
crates/utils/src/lib.rs
View file

@ -102,7 +102,7 @@ pub fn first_last_index_of<T: ::std::fmt::Debug + std::cmp::Eq>(
}
// get the path of the lib folder
// runtime dependencies like zig files, builtin_host.o are put in the lib folder
// runtime dependencies like zig files, Windows dylib builds, are put in the lib folder
pub fn get_lib_path() -> Option<PathBuf> {
let exe_relative_str_path_opt = std::env::current_exe().ok();