language-servers/deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2025-08-03 02:22:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

TS compiler refactor

Browse source 
* Compiler no longer has its own Tokio runtime. Compiler handles one
  message and then exits.

* Uses the simpler ts.CompilerHost interface instead of
  ts.LanguageServiceHost.

* avoids recompiling the same module by introducing a hacky but simple
  `hashset<string>` that stores the module names that have been already
  compiled.

* Removes the CompilerConfig op.

* Removes a lot of the mocking stuff in compiler.ts like `this._ts`. It
  is not useful as we don't even have tests.

* Turns off checkJs because it causes fmt_test to die with OOM.
This commit is contained in:
Ryan Dahl 2019-05-20 12:06:57 -04:00
parent 64d2b7bc90
commit 856c44213b
13 changed files with 491 additions and 892 deletions
Download patch file Download diff file Expand all files Collapse all files

275
cli/compiler.rs
View file

@ -1,10 +1,6 @@
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
use crate::js_errors;
use crate::js_errors::JSErrorColor;
use crate::msg; use crate::msg;
use crate::ops::op_selector_compiler;
use crate::resources; use crate::resources;
use crate::resources::ResourceId;
use crate::startup_data; use crate::startup_data;
use crate::state::*; use crate::state::*;
use crate::tokio_util; use crate::tokio_util;
@ -12,31 +8,10 @@ use crate::worker::Worker;
use deno::js_check; use deno::js_check;
use deno::Buf; use deno::Buf;
use deno::JSError; use deno::JSError;
use futures::future::*;
use futures::sync::oneshot;
use futures::Future; use futures::Future;
use futures::Stream; use futures::Stream;
use serde_json;
use std::collections::HashMap;
use std::str; use std::str;
use std::sync::atomic::AtomicUsize;
use std::sync::atomic::Ordering; use std::sync::atomic::Ordering;
use std::sync::Mutex;
use tokio::runtime::Runtime;
type CmdId = u32;
type ResponseSenderTable = HashMap<CmdId, oneshot::Sender<Buf>>;
lazy_static! {
static ref C_NEXT_CMD_ID: AtomicUsize = AtomicUsize::new(1);
// Map of response senders
static ref C_RES_SENDER_TABLE: Mutex<ResponseSenderTable> = Mutex::new(ResponseSenderTable::new());
// Shared worker resources so we can spawn
static ref C_RID: Mutex<Option<ResourceId>> = Mutex::new(None);
// tokio runtime specifically for spawning logic that is dependent on
// completetion of the compiler worker future
static ref C_RUNTIME: Mutex<Runtime> = Mutex::new(tokio_util::create_threadpool_runtime());
}
// This corresponds to JS ModuleMetaData. // This corresponds to JS ModuleMetaData.
// TODO Rename one or the other so they correspond. // TODO Rename one or the other so they correspond.
@ -72,91 +47,22 @@ impl ModuleMetaData {
} }
} }
fn new_cmd_id() -> CmdId { type CompilerConfig = Option<(String, Vec<u8>)>;
let next_rid = C_NEXT_CMD_ID.fetch_add(1, Ordering::SeqCst);
next_rid as CmdId
}
fn parse_cmd_id(res_json: &str) -> CmdId { /// Creates the JSON message send to compiler.ts's onmessage.
match serde_json::from_str::<serde_json::Value>(res_json) { fn req(root_names: Vec<String>, compiler_config: CompilerConfig) -> Buf {
Ok(serde_json::Value::Object(map)) => match map["cmdId"].as_u64() { let j = if let Some((config_path, config_data)) = compiler_config {
Some(cmd_id) => cmd_id as CmdId, json!({
_ => panic!("Error decoding compiler response: expected cmdId"), "rootNames": root_names,
}, "configPath": config_path,
_ => panic!("Error decoding compiler response"), "config": str::from_utf8(&config_data).unwrap(),
} })
} } else {
json!({
fn lazy_start(parent_state: ThreadSafeState) -> ResourceId { "rootNames": root_names,
let mut cell = C_RID.lock().unwrap(); })
cell };
.get_or_insert_with(|| { j.to_string().into_boxed_str().into_boxed_bytes()
let child_state = ThreadSafeState::new(
parent_state.flags.clone(),
parent_state.argv.clone(),
op_selector_compiler,
parent_state.progress.clone(),
);
let rid = child_state.resource.rid;
let resource = child_state.resource.clone();
let mut worker = Worker::new(
"TS".to_string(),
startup_data::compiler_isolate_init(),
child_state,
);
js_check(worker.execute("denoMain()"));
js_check(worker.execute("workerMain()"));
js_check(worker.execute("compilerMain()"));
let mut runtime = C_RUNTIME.lock().unwrap();
runtime.spawn(lazy(move || {
worker.then(move |result| -> Result<(), ()> {
// Close resource so the future created by
// handle_worker_message_stream exits
resource.close();
debug!("Compiler worker exited!");
if let Err(e) = result {
eprintln!("{}", JSErrorColor(&e).to_string());
}
std::process::exit(1);
})
}));
runtime.spawn(lazy(move || {
debug!("Start worker stream handler!");
let worker_stream = resources::get_message_stream_from_worker(rid);
worker_stream
.for_each(|msg: Buf| {
// All worker responses are handled here first before being sent via
// their respective sender. This system can be compared to the
// promise system used on the js side. This provides a way to
// resolve many futures via the same channel.
let res_json = std::str::from_utf8(&msg).unwrap();
debug!("Got message from worker: {}", res_json);
// Get the intended receiver's cmd_id from the message.
let cmd_id = parse_cmd_id(res_json);
let mut table = C_RES_SENDER_TABLE.lock().unwrap();
debug!("Cmd id for get message handler: {}", cmd_id);
// Get the corresponding response sender from the table and
// send a response.
let response_sender = table.remove(&(cmd_id as CmdId)).unwrap();
response_sender.send(msg).unwrap();
Ok(())
}).map_err(|_| ())
}));
rid
}).to_owned()
}
fn req(specifier: &str, referrer: &str, cmd_id: u32) -> Buf {
json!({
"specifier": specifier,
"referrer": referrer,
"cmdId": cmd_id,
}).to_string()
.into_boxed_str()
.into_boxed_bytes()
} }
/// Returns an optional tuple which represents the state of the compiler /// Returns an optional tuple which represents the state of the compiler
@ -165,7 +71,7 @@ fn req(specifier: &str, referrer: &str, cmd_id: u32) -> Buf {
pub fn get_compiler_config( pub fn get_compiler_config(
parent_state: &ThreadSafeState, parent_state: &ThreadSafeState,
_compiler_type: &str, _compiler_type: &str,
) -> Option<(String, Vec<u8>)> { ) -> CompilerConfig {
// The compiler type is being passed to make it easier to implement custom // The compiler type is being passed to make it easier to implement custom
// compilers in the future. // compilers in the future.
match (&parent_state.config_path, &parent_state.config) { match (&parent_state.config_path, &parent_state.config) {
@ -177,7 +83,7 @@ pub fn get_compiler_config(
} }
pub fn compile_async( pub fn compile_async(
parent_state: ThreadSafeState, state: ThreadSafeState,
specifier: &str, specifier: &str,
referrer: &str, referrer: &str,
module_meta_data: &ModuleMetaData, module_meta_data: &ModuleMetaData,
@ -186,100 +92,86 @@ pub fn compile_async(
"Running rust part of compile_sync. specifier: {}, referrer: {}", "Running rust part of compile_sync. specifier: {}, referrer: {}",
&specifier, &referrer &specifier, &referrer
); );
let cmd_id = new_cmd_id();
let req_msg = req(&specifier, &referrer, cmd_id); let root_names = vec![module_meta_data.module_name.clone()];
let compiler_config = get_compiler_config(&state, "typescript");
let req_msg = req(root_names, compiler_config);
let module_meta_data_ = module_meta_data.clone(); let module_meta_data_ = module_meta_data.clone();
let compiler_rid = lazy_start(parent_state.clone()); // Count how many times we start the compiler worker.
state.metrics.compiler_starts.fetch_add(1, Ordering::SeqCst);
let compiling_job = parent_state let mut worker = Worker::new(
"TS".to_string(),
startup_data::compiler_isolate_init(),
// TODO(ry) Maybe we should use a separate state for the compiler.
// as was done previously.
state.clone(),
);
js_check(worker.execute("denoMain()"));
js_check(worker.execute("workerMain()"));
js_check(worker.execute("compilerMain()"));
let compiling_job = state
.progress .progress
.add(format!("Compiling {}", module_meta_data_.module_name)); .add(format!("Compiling {}", module_meta_data_.module_name));
let (local_sender, local_receiver) = let resource = worker.state.resource.clone();
oneshot::channel::<Result<ModuleMetaData, Option<JSError>>>(); let compiler_rid = resource.rid;
let first_msg_fut = resources::post_message_to_worker(compiler_rid, req_msg)
.then(move |_| worker)
.then(move |result| {
if let Err(err) = result {
// TODO(ry) Need to forward the error instead of exiting.
eprintln!("{}", err.to_string());
std::process::exit(1);
}
debug!("Sent message to worker");
let stream_future =
resources::get_message_stream_from_worker(compiler_rid).into_future();
stream_future.map(|(f, _rest)| f).map_err(|(f, _rest)| f)
});
let (response_sender, response_receiver) = oneshot::channel::<Buf>(); first_msg_fut
.map_err(|_| panic!("not handled"))
.and_then(move |maybe_msg: Option<Buf>| {
let _res_msg = maybe_msg.unwrap();
// Scoping to auto dispose of locks when done using them debug!("Received message from worker");
{
let mut table = C_RES_SENDER_TABLE.lock().unwrap();
debug!("Cmd id for response sender insert: {}", cmd_id);
// Place our response sender in the table so we can find it later.
table.insert(cmd_id, response_sender);
let mut runtime = C_RUNTIME.lock().unwrap(); // TODO res is EmitResult, use serde_derive to parse it. Errors from the
runtime.spawn(lazy(move || { // worker or Diagnostics should be somehow forwarded to the caller!
resources::post_message_to_worker(compiler_rid, req_msg) // Currently they are handled inside compiler.ts with os.exit(1) and above
.then(move |_| { // with std::process::exit(1). This bad.
debug!("Sent message to worker");
response_receiver.map_err(|_| None)
}).and_then(move |res_msg| {
debug!("Received message from worker");
let res_json = std::str::from_utf8(res_msg.as_ref()).unwrap();
let res = serde_json::from_str::<serde_json::Value>(res_json)
.expect("Error decoding compiler response");
let res_data = res["data"].as_object().expect(
"Error decoding compiler response: expected object field 'data'",
);
// Explicit drop to keep reference alive until future completes. let r = state.dir.fetch_module_meta_data(
drop(compiling_job); &module_meta_data_.module_name,
".",
true,
true,
);
let module_meta_data_after_compile = r.unwrap();
match res["success"].as_bool() { // Explicit drop to keep reference alive until future completes.
Some(true) => Ok(ModuleMetaData { drop(compiling_job);
maybe_output_code: res_data["outputCode"]
.as_str()
.map(|s| s.as_bytes().to_owned()),
maybe_source_map: res_data["sourceMap"]
.as_str()
.map(|s| s.as_bytes().to_owned()),
..module_meta_data_
}),
Some(false) => {
let js_error = JSError::from_json_value(
serde_json::Value::Object(res_data.clone()),
).expect(
"Error decoding compiler response: failed to parse error",
);
Err(Some(js_errors::apply_source_map(
&js_error,
&parent_state.dir,
)))
}
_ => panic!(
"Error decoding compiler response: expected bool field 'success'"
),
}
}).then(move |result| {
local_sender.send(result).expect("Oneshot send() failed");
Ok(())
})
}));
}
local_receiver Ok(module_meta_data_after_compile)
.map_err(|e| { }).then(move |r| {
panic!( // TODO(ry) do this in worker's destructor.
"Local channel canceled before compile request could be completed: {}", // resource.close();
e r
)
}).and_then(move |result| match result {
Ok(v) => futures::future::result(Ok(v)),
Err(Some(err)) => futures::future::result(Err(err)),
Err(None) => panic!("Failed to communicate with the compiler worker."),
}) })
} }
pub fn compile_sync( pub fn compile_sync(
parent_state: ThreadSafeState, state: ThreadSafeState,
specifier: &str, specifier: &str,
referrer: &str, referrer: &str,
module_meta_data: &ModuleMetaData, module_meta_data: &ModuleMetaData,
) -> Result<ModuleMetaData, JSError> { ) -> Result<ModuleMetaData, JSError> {
tokio_util::block_on(compile_async( tokio_util::block_on(compile_async(
parent_state, state,
specifier, specifier,
referrer, referrer,
module_meta_data, module_meta_data,
@ -298,9 +190,13 @@ mod tests {
let specifier = "./tests/002_hello.ts"; let specifier = "./tests/002_hello.ts";
let referrer = cwd_string + "/"; let referrer = cwd_string + "/";
use crate::worker;
let module_name = worker::root_specifier_to_url(specifier)
.unwrap()
.to_string();
let mut out = ModuleMetaData { let mut out = ModuleMetaData {
module_name: "xxx".to_owned(), module_name,
module_redirect_source_name: None, module_redirect_source_name: None,
filename: "/tests/002_hello.ts".to_owned(), filename: "/tests/002_hello.ts".to_owned(),
media_type: msg::MediaType::TypeScript, media_type: msg::MediaType::TypeScript,
@ -322,17 +218,6 @@ mod tests {
}) })
} }
#[test]
fn test_parse_cmd_id() {
let cmd_id = new_cmd_id();
let msg = req("Hello", "World", cmd_id);
let res_json = std::str::from_utf8(&msg).unwrap();
assert_eq!(parse_cmd_id(res_json), cmd_id);
}
#[test] #[test]
fn test_get_compiler_config_no_flag() { fn test_get_compiler_config_no_flag() {
let compiler_type = "typescript"; let compiler_type = "typescript";

32
cli/msg.fbs
View file

@ -1,12 +1,13 @@
union Any { union Any {
Accept, Accept,
Cache,
Chdir, Chdir,
Chmod, Chmod,
Chown, Chown,
Close, Close,
CompilerConfig,
CompilerConfigRes,
CopyFile, CopyFile,
CreateWorker,
CreateWorkerRes,
Cwd, Cwd,
CwdRes, CwdRes,
Dial, Dial,
@ -23,6 +24,10 @@ union Any {
GlobalTimer, GlobalTimer,
GlobalTimerRes, GlobalTimerRes,
GlobalTimerStop, GlobalTimerStop,
HostGetMessage,
HostGetMessageRes,
HostGetWorkerClosed,
HostPostMessage,
IsTTY, IsTTY,
IsTTYRes, IsTTYRes,
Kill, Kill,
@ -70,12 +75,6 @@ union Any {
Symlink, Symlink,
Truncate, Truncate,
Utime, Utime,
CreateWorker,
CreateWorkerRes,
HostGetWorkerClosed,
HostGetMessage,
HostGetMessageRes,
HostPostMessage,
WorkerGetMessage, WorkerGetMessage,
WorkerGetMessageRes, WorkerGetMessageRes,
WorkerPostMessage, WorkerPostMessage,
@ -180,15 +179,6 @@ table StartRes {
xeval_delim: string; xeval_delim: string;
} }
table CompilerConfig {
compiler_type: string;
}
table CompilerConfigRes {
path: string;
data: [ubyte];
}
table FormatError { table FormatError {
error: string; error: string;
} }
@ -246,7 +236,7 @@ table FetchModuleMetaData {
table FetchModuleMetaDataRes { table FetchModuleMetaDataRes {
// If it's a non-http module, moduleName and filename will be the same. // If it's a non-http module, moduleName and filename will be the same.
// For http modules, moduleName is its resolved http URL, and filename // For http modules, module_name is its resolved http URL, and filename
// is the location of the locally downloaded source code. // is the location of the locally downloaded source code.
module_name: string; module_name: string;
filename: string; filename: string;
@ -254,6 +244,12 @@ table FetchModuleMetaDataRes {
data: [ubyte]; data: [ubyte];
} }
table Cache {
extension: string;
module_id: string;
contents: string;
}
table Chdir { table Chdir {
directory: string; directory: string;
} }

106
cli/ops.rs
View file

@ -1,7 +1,6 @@
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
use atty; use atty;
use crate::ansi; use crate::ansi;
use crate::compiler::get_compiler_config;
use crate::deno_dir::resolve_path; use crate::deno_dir::resolve_path;
use crate::dispatch_minimal::dispatch_minimal; use crate::dispatch_minimal::dispatch_minimal;
use crate::dispatch_minimal::parse_min_record; use crate::dispatch_minimal::parse_min_record;
@ -177,37 +176,30 @@ pub fn dispatch_all_legacy(
} }
} }
pub fn op_selector_compiler(inner_type: msg::Any) -> Option<OpCreator> {
match inner_type {
msg::Any::CompilerConfig => Some(op_compiler_config),
msg::Any::Cwd => Some(op_cwd),
msg::Any::FetchModuleMetaData => Some(op_fetch_module_meta_data),
msg::Any::WorkerGetMessage => Some(op_worker_get_message),
msg::Any::WorkerPostMessage => Some(op_worker_post_message),
msg::Any::Exit => Some(op_exit),
msg::Any::Start => Some(op_start),
_ => None,
}
}
/// Standard ops set for most isolates /// Standard ops set for most isolates
pub fn op_selector_std(inner_type: msg::Any) -> Option<OpCreator> { pub fn op_selector_std(inner_type: msg::Any) -> Option<OpCreator> {
match inner_type { match inner_type {
msg::Any::Accept => Some(op_accept), msg::Any::Accept => Some(op_accept),
msg::Any::Cache => Some(op_cache),
msg::Any::Chdir => Some(op_chdir), msg::Any::Chdir => Some(op_chdir),
msg::Any::Chmod => Some(op_chmod), msg::Any::Chmod => Some(op_chmod),
msg::Any::Chown => Some(op_chown), msg::Any::Chown => Some(op_chown),
msg::Any::Close => Some(op_close), msg::Any::Close => Some(op_close),
msg::Any::CopyFile => Some(op_copy_file), msg::Any::CopyFile => Some(op_copy_file),
msg::Any::CreateWorker => Some(op_create_worker),
msg::Any::Cwd => Some(op_cwd), msg::Any::Cwd => Some(op_cwd),
msg::Any::Dial => Some(op_dial), msg::Any::Dial => Some(op_dial),
msg::Any::Environ => Some(op_env), msg::Any::Environ => Some(op_env),
msg::Any::Exit => Some(op_exit), msg::Any::Exit => Some(op_exit),
msg::Any::Fetch => Some(op_fetch), msg::Any::Fetch => Some(op_fetch),
msg::Any::FetchModuleMetaData => Some(op_fetch_module_meta_data),
msg::Any::FormatError => Some(op_format_error), msg::Any::FormatError => Some(op_format_error),
msg::Any::GetRandomValues => Some(op_get_random_values), msg::Any::GetRandomValues => Some(op_get_random_values),
msg::Any::GlobalTimer => Some(op_global_timer), msg::Any::GlobalTimer => Some(op_global_timer),
msg::Any::GlobalTimerStop => Some(op_global_timer_stop), msg::Any::GlobalTimerStop => Some(op_global_timer_stop),
msg::Any::HostGetMessage => Some(op_host_get_message),
msg::Any::HostGetWorkerClosed => Some(op_host_get_worker_closed),
msg::Any::HostPostMessage => Some(op_host_post_message),
msg::Any::IsTTY => Some(op_is_tty), msg::Any::IsTTY => Some(op_is_tty),
msg::Any::Kill => Some(op_kill), msg::Any::Kill => Some(op_kill),
msg::Any::Link => Some(op_link), msg::Any::Link => Some(op_link),
@ -237,10 +229,6 @@ pub fn op_selector_std(inner_type: msg::Any) -> Option<OpCreator> {
msg::Any::Symlink => Some(op_symlink), msg::Any::Symlink => Some(op_symlink),
msg::Any::Truncate => Some(op_truncate), msg::Any::Truncate => Some(op_truncate),
msg::Any::Utime => Some(op_utime), msg::Any::Utime => Some(op_utime),
msg::Any::CreateWorker => Some(op_create_worker),
msg::Any::HostGetWorkerClosed => Some(op_host_get_worker_closed),
msg::Any::HostGetMessage => Some(op_host_get_message),
msg::Any::HostPostMessage => Some(op_host_post_message),
msg::Any::Write => Some(op_write), msg::Any::Write => Some(op_write),
// TODO(ry) split these out so that only the appropriate Workers can access // TODO(ry) split these out so that only the appropriate Workers can access
@ -446,6 +434,53 @@ pub fn odd_future(err: DenoError) -> Box<OpWithError> {
Box::new(futures::future::err(err)) Box::new(futures::future::err(err))
} }
fn op_cache(
state: &ThreadSafeState,
base: &msg::Base<'_>,
data: Option<PinnedBuf>,
) -> Box<OpWithError> {
assert!(data.is_none());
let inner = base.inner_as_cache().unwrap();
let extension = inner.extension().unwrap();
let module_id = inner.module_id().unwrap();
let contents = inner.contents().unwrap();
state.mark_compiled(&module_id);
// TODO It shouldn't be necessary to call fetch_module_meta_data() here.
// However, we need module_meta_data.source_code in order to calculate the
// cache path. In the future, checksums will not be used in the cache
// filenames and this requirement can be removed. See
// https://github.com/denoland/deno/issues/2057
let r = state.dir.fetch_module_meta_data(module_id, ".", true, true);
if let Err(err) = r {
return odd_future(err);
}
let module_meta_data = r.unwrap();
let (js_cache_path, source_map_path) = state
.dir
.cache_path(&module_meta_data.filename, &module_meta_data.source_code);
if extension == ".map" {
debug!("cache {:?}", source_map_path);
let r = fs::write(source_map_path, contents);
if let Err(err) = r {
return odd_future(err.into());
}
} else if extension == ".js" {
debug!("cache {:?}", js_cache_path);
let r = fs::write(js_cache_path, contents);
if let Err(err) = r {
return odd_future(err.into());
}
} else {
unreachable!();
}
ok_future(empty_buf())
}
// https://github.com/denoland/deno/blob/golang/os.go#L100-L154 // https://github.com/denoland/deno/blob/golang/os.go#L100-L154
fn op_fetch_module_meta_data( fn op_fetch_module_meta_data(
state: &ThreadSafeState, state: &ThreadSafeState,
@ -498,41 +533,6 @@ fn op_fetch_module_meta_data(
Box::new(futures::future::result(tokio_util::block_on(fut))) Box::new(futures::future::result(tokio_util::block_on(fut)))
} }
/// Retrieve any relevant compiler configuration.
fn op_compiler_config(
state: &ThreadSafeState,
base: &msg::Base<'_>,
data: Option<PinnedBuf>,
) -> Box<OpWithError> {
assert!(data.is_none());
let inner = base.inner_as_compiler_config().unwrap();
let cmd_id = base.cmd_id();
let compiler_type = inner.compiler_type().unwrap();
Box::new(futures::future::result(|| -> OpResult {
let builder = &mut FlatBufferBuilder::new();
let (path, out) = match get_compiler_config(state, compiler_type) {
Some(val) => val,
_ => ("".to_owned(), vec![]),
};
let data_off = builder.create_vector(&out);
let msg_args = msg::CompilerConfigResArgs {
path: Some(builder.create_string(&path)),
data: Some(data_off),
};
let inner = msg::CompilerConfigRes::create(builder, &msg_args);
Ok(serialize_response(
cmd_id,
builder,
msg::BaseArgs {
inner: Some(inner.as_union_value()),
inner_type: msg::Any::CompilerConfigRes,
..Default::default()
},
))
}()))
}
fn op_chdir( fn op_chdir(
_state: &ThreadSafeState, _state: &ThreadSafeState,
base: &msg::Base<'_>, base: &msg::Base<'_>,

18
cli/state.rs
View file

@ -15,6 +15,7 @@ use deno::PinnedBuf;
use futures::future::Shared; use futures::future::Shared;
use std; use std;
use std::collections::HashMap; use std::collections::HashMap;
use std::collections::HashSet;
use std::env; use std::env;
use std::fs; use std::fs;
use std::ops::Deref; use std::ops::Deref;
@ -37,6 +38,7 @@ pub struct Metrics {
pub bytes_sent_data: AtomicUsize, pub bytes_sent_data: AtomicUsize,
pub bytes_received: AtomicUsize, pub bytes_received: AtomicUsize,
pub resolve_count: AtomicUsize, pub resolve_count: AtomicUsize,
pub compiler_starts: AtomicUsize,
} }
/// Isolate cannot be passed between threads but ThreadSafeState can. /// Isolate cannot be passed between threads but ThreadSafeState can.
@ -66,6 +68,11 @@ pub struct State {
pub dispatch_selector: ops::OpSelector, pub dispatch_selector: ops::OpSelector,
/// Reference to global progress bar. /// Reference to global progress bar.
pub progress: Progress, pub progress: Progress,
/// Set of all URLs that have been compiled. This is a hacky way to work
/// around the fact that --reload will force multiple compilations of the same
/// module.
compiled: Mutex<HashSet<String>>,
} }
impl Clone for ThreadSafeState { impl Clone for ThreadSafeState {
@ -157,6 +164,7 @@ impl ThreadSafeState {
resource, resource,
dispatch_selector, dispatch_selector,
progress, progress,
compiled: Mutex::new(HashSet::new()),
})) }))
} }
@ -177,6 +185,16 @@ impl ThreadSafeState {
} }
} }
pub fn mark_compiled(&self, module_id: &str) {
let mut c = self.compiled.lock().unwrap();
c.insert(module_id.to_string());
}
pub fn has_compiled(&self, module_id: &str) -> bool {
let c = self.compiled.lock().unwrap();
c.contains(module_id)
}
#[inline] #[inline]
pub fn check_read(&self, filename: &str) -> DenoResult<()> { pub fn check_read(&self, filename: &str) -> DenoResult<()> {
self.permissions.check_read(filename) self.permissions.check_read(filename)

88
cli/worker.rs
View file

@ -212,35 +212,39 @@ fn fetch_module_meta_data_and_maybe_compile_async(
specifier: &str, specifier: &str,
referrer: &str, referrer: &str,
) -> impl Future<Item = ModuleMetaData, Error = DenoError> { ) -> impl Future<Item = ModuleMetaData, Error = DenoError> {
let use_cache = !state.flags.reload;
let no_fetch = state.flags.no_fetch;
let state_ = state.clone(); let state_ = state.clone();
let specifier = specifier.to_string(); let specifier = specifier.to_string();
let referrer = referrer.to_string(); let referrer = referrer.to_string();
state
.dir let f = futures::future::result(Worker::resolve(&specifier, &referrer));
.fetch_module_meta_data_async(&specifier, &referrer, use_cache, no_fetch) f.and_then(move |module_id| {
.and_then(move |out| { let use_cache = !state_.flags.reload || state_.has_compiled(&module_id);
if out.media_type == msg::MediaType::TypeScript let no_fetch = state_.flags.no_fetch;
&& !out.has_output_code_and_source_map()
{ state_
debug!(">>>>> compile_sync START"); .dir
Either::A( .fetch_module_meta_data_async(&specifier, &referrer, use_cache, no_fetch)
compile_async(state_.clone(), &specifier, &referrer, &out) .and_then(move |out| {
.map_err(|e| { if out.media_type == msg::MediaType::TypeScript
debug!("compiler error exiting!"); && !out.has_output_code_and_source_map()
eprintln!("{}", JSErrorColor(&e).to_string()); {
std::process::exit(1); debug!(">>>>> compile_sync START");
}).and_then(move |out| { Either::A(
debug!(">>>>> compile_sync END"); compile_async(state_.clone(), &specifier, &referrer, &out)
state_.dir.code_cache(&out)?; .map_err(|e| {
Ok(out) debug!("compiler error exiting!");
}), eprintln!("{}", JSErrorColor(&e).to_string());
) std::process::exit(1);
} else { }).and_then(move |out| {
Either::B(futures::future::ok(out)) debug!(">>>>> compile_sync END");
} Ok(out)
}) }),
)
} else {
Either::B(futures::future::ok(out))
}
})
})
} }
pub fn fetch_module_meta_data_and_maybe_compile( pub fn fetch_module_meta_data_and_maybe_compile(
@ -297,6 +301,8 @@ mod tests {
let metrics = &state_.metrics; let metrics = &state_.metrics;
assert_eq!(metrics.resolve_count.load(Ordering::SeqCst), 2); assert_eq!(metrics.resolve_count.load(Ordering::SeqCst), 2);
// Check that we didn't start the compiler.
assert_eq!(metrics.compiler_starts.load(Ordering::SeqCst), 0);
} }
#[test] #[test]
@ -327,6 +333,8 @@ mod tests {
let metrics = &state_.metrics; let metrics = &state_.metrics;
assert_eq!(metrics.resolve_count.load(Ordering::SeqCst), 2); assert_eq!(metrics.resolve_count.load(Ordering::SeqCst), 2);
// Check that we didn't start the compiler.
assert_eq!(metrics.compiler_starts.load(Ordering::SeqCst), 0);
} }
#[test] #[test]
@ -361,6 +369,8 @@ mod tests {
let metrics = &state_.metrics; let metrics = &state_.metrics;
assert_eq!(metrics.resolve_count.load(Ordering::SeqCst), 3); assert_eq!(metrics.resolve_count.load(Ordering::SeqCst), 3);
// Check that we've only invoked the compiler once.
assert_eq!(metrics.compiler_starts.load(Ordering::SeqCst), 1);
} }
fn create_test_worker() -> Worker { fn create_test_worker() -> Worker {
@ -459,20 +469,24 @@ mod tests {
#[test] #[test]
fn execute_mod_resolve_error() { fn execute_mod_resolve_error() {
// "foo" is not a vailid module specifier so this should return an error. tokio_util::init(|| {
let worker = create_test_worker(); // "foo" is not a vailid module specifier so this should return an error.
let js_url = root_specifier_to_url("does-not-exist").unwrap(); let worker = create_test_worker();
let result = worker.execute_mod_async(&js_url, false).wait(); let js_url = root_specifier_to_url("does-not-exist").unwrap();
assert!(result.is_err()); let result = worker.execute_mod_async(&js_url, false).wait();
assert!(result.is_err());
})
} }
#[test] #[test]
fn execute_mod_002_hello() { fn execute_mod_002_hello() {
// This assumes cwd is project root (an assumption made throughout the tokio_util::init(|| {
// tests). // This assumes cwd is project root (an assumption made throughout the
let worker = create_test_worker(); // tests).
let js_url = root_specifier_to_url("./tests/002_hello.ts").unwrap(); let worker = create_test_worker();
let result = worker.execute_mod_async(&js_url, false).wait(); let js_url = root_specifier_to_url("./tests/002_hello.ts").unwrap();
assert!(result.is_ok()); let result = worker.execute_mod_async(&js_url, false).wait();
assert!(result.is_ok());
})
} }
} }

2
core/shared_queue.js
View file

@ -109,7 +109,7 @@ SharedQueue Binary Layout
let end = off + buf.byteLength; let end = off + buf.byteLength;
let index = numRecords(); let index = numRecords();
if (end > shared32.byteLength || index >= MAX_RECORDS) { if (end > shared32.byteLength || index >= MAX_RECORDS) {
console.log("shared_queue.ts push fail"); // console.log("shared_queue.js push fail");
return false; return false;
} }
setEnd(index, end); setEnd(index, end);

796
js/compiler.ts
View file

@ -1,76 +1,40 @@
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
import * as ts from "typescript";
import * as msg from "gen/cli/msg_generated"; import * as msg from "gen/cli/msg_generated";
import { window } from "./window";
import { assetSourceCode } from "./assets";
import { bold, cyan, yellow } from "./colors";
import { Console } from "./console";
import { core } from "./core"; import { core } from "./core";
import { cwd } from "./dir";
import { sendSync } from "./dispatch";
import * as flatbuffers from "./flatbuffers"; import * as flatbuffers from "./flatbuffers";
import { sendSync } from "./dispatch";
import { TextDecoder } from "./text_encoding";
import * as ts from "typescript";
import * as os from "./os"; import * as os from "./os";
import { TextDecoder, TextEncoder } from "./text_encoding"; import { bold, cyan, yellow } from "./colors";
import { clearTimer, setTimeout } from "./timers"; import { window } from "./window";
import { postMessage, workerClose, workerMain } from "./workers"; import { postMessage, workerClose, workerMain } from "./workers";
import { assert, log, notImplemented } from "./util"; import { Console } from "./console";
import { assert, notImplemented } from "./util";
import * as util from "./util";
import { cwd } from "./dir";
import { assetSourceCode } from "./assets";
const EOL = "\n"; // Startup boilerplate. This is necessary because the compiler has its own
const ASSETS = "$asset$"; // snapshot. (It would be great if we could remove these things or centralize
const LIB_RUNTIME = `${ASSETS}/lib.deno_runtime.d.ts`; // them somewhere else.)
// An instance of console
const console = new Console(core.print); const console = new Console(core.print);
window.console = console;
window.workerMain = workerMain;
export default function denoMain(): void {
os.start("TS");
}
/** The location that a module is being loaded from. This could be a directory, const ASSETS = "$asset$";
* like `.`, or it could be a module specifier like const OUT_DIR = "$deno$";
* `http://gist.github.com/somefile.ts`
*/
type ContainingFile = string;
/** The internal local filename of a compiled module. It will often be something
* like `/home/ry/.deno/gen/f7b4605dfbc4d3bb356e98fda6ceb1481e4a8df5.js`
*/
type ModuleFileName = string;
/** The original resolved resource name.
* Path to cached module file or URL from which dependency was retrieved
*/
type ModuleId = string;
/** The external name of a module - could be a URL or could be a relative path.
* Examples `http://gist.github.com/somefile.ts` or `./somefile.ts`
*/
type ModuleSpecifier = string;
/** The compiled source code which is cached in `.deno/gen/` */
type OutputCode = string;
/** The original source code */
type SourceCode = string;
/** The output source map */
type SourceMap = string;
/** The format of the work message payload coming from the privileged side */ /** The format of the work message payload coming from the privileged side */
interface CompilerLookup { interface CompilerReq {
specifier: ModuleSpecifier; rootNames: string[];
referrer: ContainingFile; // TODO(ry) add compiler config to this interface.
cmdId: number; // options: ts.CompilerOptions;
} configPath?: string;
config?: string;
/** Abstraction of the APIs required from the `os` module so they can be
* easily mocked.
*/
interface Os {
fetchModuleMetaData: typeof os.fetchModuleMetaData;
exit: typeof os.exit;
noColor: typeof os.noColor;
}
/** Abstraction of the APIs required from the `typescript` module so they can
* be easily mocked.
*/
interface Ts {
convertCompilerOptionsFromJson: typeof ts.convertCompilerOptionsFromJson;
createLanguageService: typeof ts.createLanguageService;
formatDiagnosticsWithColorAndContext: typeof ts.formatDiagnosticsWithColorAndContext;
formatDiagnostics: typeof ts.formatDiagnostics;
parseConfigFileTextToJson: typeof ts.parseConfigFileTextToJson;
} }
/** Options that either do nothing in Deno, or would cause undesired behavior /** Options that either do nothing in Deno, or would cause undesired behavior
@ -134,48 +98,68 @@ const ignoredCompilerOptions: ReadonlyArray<string> = [
"watch" "watch"
]; ];
/** A simple object structure for caching resolved modules and their contents. interface ModuleMetaData {
* moduleName: string | undefined;
* Named `ModuleMetaData` to clarify it is just a representation of meta data of filename: string | undefined;
* the module, not the actual module instance. mediaType: msg.MediaType;
*/ sourceCode: string | undefined;
class ModuleMetaData implements ts.IScriptSnapshot { }
public scriptVersion = "";
constructor( function fetchModuleMetaData(
public readonly moduleId: ModuleId, specifier: string,
public readonly fileName: ModuleFileName, referrer: string
public readonly mediaType: msg.MediaType, ): ModuleMetaData {
public readonly sourceCode: SourceCode = "", util.log("compiler.fetchModuleMetaData", { specifier, referrer });
public outputCode: OutputCode = "", // Send FetchModuleMetaData message
public sourceMap: SourceMap = "" const builder = flatbuffers.createBuilder();
) { const specifier_ = builder.createString(specifier);
if (outputCode !== "" || fileName.endsWith(".d.ts")) { const referrer_ = builder.createString(referrer);
this.scriptVersion = "1"; const inner = msg.FetchModuleMetaData.createFetchModuleMetaData(
} builder,
} specifier_,
referrer_
);
const baseRes = sendSync(builder, msg.Any.FetchModuleMetaData, inner);
assert(baseRes != null);
assert(
msg.Any.FetchModuleMetaDataRes === baseRes!.innerType(),
`base.innerType() unexpectedly is ${baseRes!.innerType()}`
);
const fetchModuleMetaDataRes = new msg.FetchModuleMetaDataRes();
assert(baseRes!.inner(fetchModuleMetaDataRes) != null);
const dataArray = fetchModuleMetaDataRes.dataArray();
const decoder = new TextDecoder();
const sourceCode = dataArray ? decoder.decode(dataArray) : undefined;
// flatbuffers returns `null` for an empty value, this does not fit well with
// idiomatic TypeScript under strict null checks, so converting to `undefined`
return {
moduleName: fetchModuleMetaDataRes.moduleName() || undefined,
filename: fetchModuleMetaDataRes.filename() || undefined,
mediaType: fetchModuleMetaDataRes.mediaType(),
sourceCode
};
}
/** TypeScript IScriptSnapshot Interface */ /** For caching source map and compiled js */
function cache(extension: string, moduleId: string, contents: string): void {
public getText(start: number, end: number): string { util.log("compiler.cache", moduleId);
return start === 0 && end === this.sourceCode.length const builder = flatbuffers.createBuilder();
? this.sourceCode const extension_ = builder.createString(extension);
: this.sourceCode.substring(start, end); const moduleId_ = builder.createString(moduleId);
} const contents_ = builder.createString(contents);
const inner = msg.Cache.createCache(
public getLength(): number { builder,
return this.sourceCode.length; extension_,
} moduleId_,
contents_
public getChangeRange(): undefined { );
// Required `IScriptSnapshot` API, but not implemented/needed in deno const baseRes = sendSync(builder, msg.Any.Cache, inner);
return undefined; assert(baseRes == null);
}
} }
/** Returns the TypeScript Extension enum for a given media type. */ /** Returns the TypeScript Extension enum for a given media type. */
function getExtension( function getExtension(
fileName: ModuleFileName, fileName: string,
mediaType: msg.MediaType mediaType: msg.MediaType
): ts.Extension { ): ts.Extension {
switch (mediaType) { switch (mediaType) {
@ -191,294 +175,34 @@ function getExtension(
} }
} }
/** Generate output code for a provided JSON string along with its source. */ class Host implements ts.CompilerHost {
function jsonEsmTemplate(
jsonString: string,
sourceFileName: string
): OutputCode {
return (
`const _json = JSON.parse(\`${jsonString}\`);\n` +
`export default _json;\n` +
`//# sourceURL=${sourceFileName}\n`
);
}
/** A singleton class that combines the TypeScript Language Service host API
* with Deno specific APIs to provide an interface for compiling and running
* TypeScript and JavaScript modules.
*/
class Compiler implements ts.LanguageServiceHost, ts.FormatDiagnosticsHost {
// Modules are usually referenced by their ModuleSpecifier and ContainingFile,
// and keeping a map of the resolved module file name allows more efficient
// future resolution
private readonly _fileNamesMap = new Map<
ContainingFile,
Map<ModuleSpecifier, ModuleFileName>
>();
// A reference to the log utility, so it can be monkey patched during testing
private _log = log;
// A map of module file names to module meta data
private readonly _moduleMetaDataMap = new Map<
ModuleFileName,
ModuleMetaData
>();
// TODO ideally this are not static and can be influenced by command line
// arguments
private readonly _options: ts.CompilerOptions = { private readonly _options: ts.CompilerOptions = {
allowJs: true, allowJs: true,
allowNonTsExtensions: true, allowNonTsExtensions: true,
checkJs: true, checkJs: false,
esModuleInterop: true, esModuleInterop: true,
module: ts.ModuleKind.ESNext, module: ts.ModuleKind.ESNext,
outDir: "$deno$", outDir: OUT_DIR,
resolveJsonModule: true, resolveJsonModule: true,
sourceMap: true, sourceMap: true,
stripComments: true, stripComments: true,
target: ts.ScriptTarget.ESNext target: ts.ScriptTarget.ESNext
}; };
// A reference to the `./os.ts` module, so it can be monkey patched during
// testing
private _os: Os = os;
// Used to contain the script file we are currently running
private _scriptFileNames: string[] = [];
// A reference to the TypeScript LanguageService instance so it can be
// monkey patched during testing
private _service: ts.LanguageService;
// A reference to `typescript` module so it can be monkey patched during
// testing
private _ts: Ts = ts;
private readonly _assetsSourceCode: { [key: string]: string };
/** The TypeScript language service often refers to the resolved fileName of
* a module, this is a shortcut to avoid unnecessary module resolution logic
* for modules that may have been initially resolved by a `moduleSpecifier`
* and `containingFile`. Also, `resolveModule()` throws when the module
* cannot be resolved, which isn't always valid when dealing with the
* TypeScript compiler, but the TypeScript compiler shouldn't be asking about
* external modules that we haven't told it about yet.
*/
private _getModuleMetaData(
fileName: ModuleFileName
): ModuleMetaData | undefined {
return (
this._moduleMetaDataMap.get(fileName) ||
(fileName.startsWith(ASSETS)
? this._resolveModule(fileName, "")
: undefined)
);
}
/** Log TypeScript diagnostics to the console and exit */
private _logDiagnostics(diagnostics: ts.Diagnostic[]): never {
const errMsg = this._os.noColor
? this._ts.formatDiagnostics(diagnostics, this)
: this._ts.formatDiagnosticsWithColorAndContext(diagnostics, this);
console.log(errMsg);
// TODO The compiler isolate shouldn't exit. Errors should be forwarded to
// to the caller and the caller exit.
return this._os.exit(1);
}
/** Given a `moduleSpecifier` and `containingFile` retrieve the cached
* `fileName` for a given module. If the module has yet to be resolved
* this will return `undefined`.
*/
private _resolveFileName(
moduleSpecifier: ModuleSpecifier,
containingFile: ContainingFile
): ModuleFileName | undefined {
this._log("compiler._resolveFileName", { moduleSpecifier, containingFile });
const innerMap = this._fileNamesMap.get(containingFile);
if (innerMap) {
return innerMap.get(moduleSpecifier);
}
return undefined;
}
/** Given a `moduleSpecifier` and `containingFile`, resolve the module and
* return the `ModuleMetaData`.
*/
private _resolveModule(
moduleSpecifier: ModuleSpecifier,
containingFile: ContainingFile
): ModuleMetaData {
this._log("compiler._resolveModule", { moduleSpecifier, containingFile });
assert(moduleSpecifier != null && moduleSpecifier.length > 0);
let fileName = this._resolveFileName(moduleSpecifier, containingFile);
if (fileName && this._moduleMetaDataMap.has(fileName)) {
return this._moduleMetaDataMap.get(fileName)!;
}
let moduleId: ModuleId | undefined;
let mediaType = msg.MediaType.Unknown;
let sourceCode: SourceCode | undefined;
if (
moduleSpecifier.startsWith(ASSETS) ||
containingFile.startsWith(ASSETS)
) {
// Assets are compiled into the runtime javascript bundle.
// we _know_ `.pop()` will return a string, but TypeScript doesn't so
// not null assertion
moduleId = moduleSpecifier.split("/").pop()!;
const assetName = moduleId.includes(".") ? moduleId : `${moduleId}.d.ts`;
assert(
assetName in this._assetsSourceCode,
`No such asset "${assetName}"`
);
mediaType = msg.MediaType.TypeScript;
sourceCode = this._assetsSourceCode[assetName];
fileName = `${ASSETS}/${assetName}`;
} else {
// We query Rust with a CodeFetch message. It will load the sourceCode,
// and if there is any outputCode cached, will return that as well.
const fetchResponse = this._os.fetchModuleMetaData(
moduleSpecifier,
containingFile
);
moduleId = fetchResponse.moduleName;
fileName = fetchResponse.filename;
mediaType = fetchResponse.mediaType;
sourceCode = fetchResponse.sourceCode;
}
assert(moduleId != null, "No module ID.");
assert(fileName != null, "No file name.");
assert(
mediaType !== msg.MediaType.Unknown,
`Unknown media type for: "${moduleSpecifier}" from "${containingFile}".`
);
this._log(
"resolveModule sourceCode length:",
sourceCode && sourceCode.length
);
this._log("resolveModule has media type:", msg.MediaType[mediaType]);
// fileName is asserted above, but TypeScript does not track so not null
this._setFileName(moduleSpecifier, containingFile, fileName!);
if (fileName && this._moduleMetaDataMap.has(fileName)) {
return this._moduleMetaDataMap.get(fileName)!;
}
const moduleMetaData = new ModuleMetaData(
moduleId!,
fileName!,
mediaType,
sourceCode
);
this._moduleMetaDataMap.set(fileName!, moduleMetaData);
return moduleMetaData;
}
/** Caches the resolved `fileName` in relationship to the `moduleSpecifier`
* and `containingFile` in order to reduce calls to the privileged side
* to retrieve the contents of a module.
*/
private _setFileName(
moduleSpecifier: ModuleSpecifier,
containingFile: ContainingFile,
fileName: ModuleFileName
): void {
this._log("compiler._setFileName", { moduleSpecifier, containingFile });
let innerMap = this._fileNamesMap.get(containingFile);
if (!innerMap) {
innerMap = new Map();
this._fileNamesMap.set(containingFile, innerMap);
}
innerMap.set(moduleSpecifier, fileName);
}
constructor(assetsSourceCode: { [key: string]: string }) {
this._assetsSourceCode = assetsSourceCode;
this._service = this._ts.createLanguageService(this);
}
// Deno specific compiler API
/** Retrieve the output of the TypeScript compiler for a given module.
*/
compile(
moduleSpecifier: ModuleSpecifier,
containingFile: ContainingFile
): { outputCode: OutputCode; sourceMap: SourceMap } {
this._log("compiler.compile", { moduleSpecifier, containingFile });
const moduleMetaData = this._resolveModule(moduleSpecifier, containingFile);
const { fileName, mediaType, sourceCode } = moduleMetaData;
this._scriptFileNames = [fileName];
let outputCode: string;
let sourceMap = "";
// Instead of using TypeScript to transpile JSON modules, we will just do
// it directly.
if (mediaType === msg.MediaType.Json) {
outputCode = moduleMetaData.outputCode = jsonEsmTemplate(
sourceCode,
fileName
);
} else {
const service = this._service;
assert(
mediaType === msg.MediaType.TypeScript ||
mediaType === msg.MediaType.JavaScript
);
const output = service.getEmitOutput(fileName);
// Get the relevant diagnostics - this is 3x faster than
// `getPreEmitDiagnostics`.
const diagnostics = [
// TypeScript is overly opinionated that only CommonJS modules kinds can
// support JSON imports. Allegedly this was fixed in
// Microsoft/TypeScript#26825 but that doesn't seem to be working here,
// so we will ignore complaints about this compiler setting.
...service
.getCompilerOptionsDiagnostics()
.filter((diagnostic): boolean => diagnostic.code !== 5070),
...service.getSyntacticDiagnostics(fileName),
...service.getSemanticDiagnostics(fileName)
];
if (diagnostics.length > 0) {
this._logDiagnostics(diagnostics);
}
assert(
!output.emitSkipped,
"The emit was skipped for an unknown reason."
);
assert(
output.outputFiles.length === 2,
`Expected 2 files to be emitted, got ${output.outputFiles.length}.`
);
const [sourceMapFile, outputFile] = output.outputFiles;
assert(
sourceMapFile.name.endsWith(".map"),
"Expected first emitted file to be a source map"
);
assert(
outputFile.name.endsWith(".js"),
"Expected second emitted file to be JavaScript"
);
outputCode = moduleMetaData.outputCode = `${
outputFile.text
}\n//# sourceURL=${fileName}`;
sourceMap = moduleMetaData.sourceMap = sourceMapFile.text;
}
moduleMetaData.scriptVersion = "1";
return { outputCode, sourceMap };
}
/** Take a configuration string, parse it, and use it to merge with the /** Take a configuration string, parse it, and use it to merge with the
* compiler's configuration options. The method returns an array of compiler * compiler's configuration options. The method returns an array of compiler
* options which were ignored, or `undefined`. * options which were ignored, or `undefined`.
*/ */
configure(path: string, configurationText: string): string[] | undefined { configure(path: string, configurationText: string): string[] | undefined {
this._log("compile.configure", path); util.log("compile.configure", path);
const { config, error } = this._ts.parseConfigFileTextToJson( const { config, error } = ts.parseConfigFileTextToJson(
path, path,
configurationText configurationText
); );
if (error) { if (error) {
this._logDiagnostics([error]); this._logDiagnostics([error]);
} }
const { options, errors } = this._ts.convertCompilerOptionsFromJson( const { options, errors } = ts.convertCompilerOptionsFromJson(
config.compilerOptions, config.compilerOptions,
cwd() cwd()
); );
@ -499,195 +223,201 @@ class Compiler implements ts.LanguageServiceHost, ts.FormatDiagnosticsHost {
return ignoredOptions.length ? ignoredOptions : undefined; return ignoredOptions.length ? ignoredOptions : undefined;
} }
// TypeScript Language Service and Format Diagnostic Host API
getCanonicalFileName(fileName: string): string {
this._log("getCanonicalFileName", fileName);
return fileName;
}
getCompilationSettings(): ts.CompilerOptions { getCompilationSettings(): ts.CompilerOptions {
this._log("getCompilationSettings()"); util.log("getCompilationSettings()");
return this._options; return this._options;
} }
getNewLine(): string { /** Log TypeScript diagnostics to the console and exit */
return EOL; _logDiagnostics(diagnostics: ReadonlyArray<ts.Diagnostic>): never {
const errMsg = os.noColor
? ts.formatDiagnostics(diagnostics, this)
: ts.formatDiagnosticsWithColorAndContext(diagnostics, this);
console.log(errMsg);
// TODO The compiler isolate shouldn't call os.exit(). (In fact, it
// shouldn't even have access to call that op.) Errors should be forwarded
// to to the caller and the caller exit.
return os.exit(1);
} }
getScriptFileNames(): string[] { fileExists(_fileName: string): boolean {
// This is equal to `"files"` in the `tsconfig.json`, therefore we only need
// to include the actual base source files we are evaluating at the moment,
// which would be what is set during the `.compile()`
return this._scriptFileNames;
}
getScriptKind(fileName: ModuleFileName): ts.ScriptKind {
this._log("getScriptKind()", fileName);
const moduleMetaData = this._getModuleMetaData(fileName);
if (moduleMetaData) {
switch (moduleMetaData.mediaType) {
case msg.MediaType.TypeScript:
return ts.ScriptKind.TS;
case msg.MediaType.JavaScript:
return ts.ScriptKind.JS;
case msg.MediaType.Json:
return ts.ScriptKind.JSON;
default:
return this._options.allowJs ? ts.ScriptKind.JS : ts.ScriptKind.TS;
}
} else {
return this._options.allowJs ? ts.ScriptKind.JS : ts.ScriptKind.TS;
}
}
getScriptVersion(fileName: ModuleFileName): string {
const moduleMetaData = this._getModuleMetaData(fileName);
const version = (moduleMetaData && moduleMetaData.scriptVersion) || "";
this._log("getScriptVersion()", fileName, version);
return version;
}
getScriptSnapshot(fileName: ModuleFileName): ts.IScriptSnapshot | undefined {
this._log("getScriptSnapshot()", fileName);
return this._getModuleMetaData(fileName);
}
getCurrentDirectory(): string {
this._log("getCurrentDirectory()");
return "";
}
getDefaultLibFileName(): string {
this._log("getDefaultLibFileName()");
const moduleSpecifier = LIB_RUNTIME;
const moduleMetaData = this._getModuleMetaData(moduleSpecifier);
assert(moduleMetaData != null);
return moduleMetaData!.fileName;
}
useCaseSensitiveFileNames(): boolean {
this._log("useCaseSensitiveFileNames()");
return true;
}
readFile(path: string): string | undefined {
this._log("readFile()", path);
return notImplemented(); return notImplemented();
} }
fileExists(fileName: string): boolean { readFile(_fileName: string): string | undefined {
const moduleMetaData = this._getModuleMetaData(fileName); return notImplemented();
const exists = moduleMetaData != null; }
this._log("fileExists()", fileName, exists);
return exists; getSourceFile(
fileName: string,
languageVersion: ts.ScriptTarget,
onError?: (message: string) => void,
shouldCreateNewSourceFile?: boolean
): ts.SourceFile | undefined {
assert(!shouldCreateNewSourceFile);
util.log("getSourceFile", fileName);
const moduleMetaData = this._resolveModule(fileName, ".");
if (!moduleMetaData || !moduleMetaData.sourceCode) {
return undefined;
}
return ts.createSourceFile(
fileName,
moduleMetaData.sourceCode,
languageVersion
);
}
getDefaultLibFileName(_options: ts.CompilerOptions): string {
return ASSETS + "/lib.deno_runtime.d.ts";
}
writeFile(
fileName: string,
data: string,
writeByteOrderMark: boolean,
onError?: (message: string) => void,
sourceFiles?: ReadonlyArray<ts.SourceFile>
): void {
util.log("writeFile", fileName);
assert(sourceFiles != null && sourceFiles.length == 1);
const sourceFileName = sourceFiles![0].fileName;
if (fileName.endsWith(".map")) {
// Source Map
cache(".map", sourceFileName, data);
} else if (fileName.endsWith(".js") || fileName.endsWith(".json")) {
// Compiled JavaScript
cache(".js", sourceFileName, data);
} else {
assert(false, "Trying to cache unhandled file type " + fileName);
}
}
getCurrentDirectory(): string {
return "";
}
getCanonicalFileName(fileName: string): string {
// console.log("getCanonicalFileName", fileName);
return fileName;
}
useCaseSensitiveFileNames(): boolean {
return true;
}
getNewLine(): string {
return "\n";
} }
resolveModuleNames( resolveModuleNames(
moduleNames: ModuleSpecifier[], moduleNames: string[],
containingFile: ContainingFile containingFile: string
): Array<ts.ResolvedModuleFull | ts.ResolvedModule> { ): Array<ts.ResolvedModuleFull | undefined> {
this._log("resolveModuleNames()", { moduleNames, containingFile }); util.log("resolveModuleNames()", { moduleNames, containingFile });
const resolvedModuleNames: ts.ResolvedModuleFull[] = []; return moduleNames.map(
for (const moduleName of moduleNames) { (moduleName): ts.ResolvedModuleFull | undefined => {
const moduleMetaData = this._resolveModule(moduleName, containingFile); const moduleMetaData = this._resolveModule(moduleName, containingFile);
// According to the interface we shouldn't return `undefined` but if we if (moduleMetaData.moduleName) {
// fail to return the same length of modules to those we cannot resolve const resolvedFileName = moduleMetaData.moduleName;
// then TypeScript fails on an assertion that the lengths can't be // This flags to the compiler to not go looking to transpile functional
// different, so we have to return an "empty" resolved module // code, anything that is in `/$asset$/` is just library code
// TODO: all this does is push the problem downstream, and TypeScript const isExternalLibraryImport = moduleName.startsWith(ASSETS);
// will complain it can't identify the type of the file and throw const r = {
// a runtime exception, so we need to handle missing modules better resolvedFileName,
const resolvedFileName = moduleMetaData.fileName || ""; isExternalLibraryImport,
// This flags to the compiler to not go looking to transpile functional extension: getExtension(resolvedFileName, moduleMetaData.mediaType)
// code, anything that is in `/$asset$/` is just library code };
const isExternalLibraryImport = resolvedFileName.startsWith(ASSETS); return r;
resolvedModuleNames.push({ } else {
resolvedFileName, return undefined;
isExternalLibraryImport, }
extension: getExtension(resolvedFileName, moduleMetaData.mediaType) }
}); );
}
private _resolveModule(specifier: string, referrer: string): ModuleMetaData {
// Handle built-in assets specially.
if (specifier.startsWith(ASSETS)) {
const moduleName = specifier.split("/").pop()!;
const assetName = moduleName.includes(".")
? moduleName
: `${moduleName}.d.ts`;
assert(assetName in assetSourceCode, `No such asset "${assetName}"`);
const sourceCode = assetSourceCode[assetName];
return {
moduleName,
filename: specifier,
mediaType: msg.MediaType.TypeScript,
sourceCode
};
} }
return resolvedModuleNames; return fetchModuleMetaData(specifier, referrer);
} }
} }
const compiler = new Compiler(assetSourceCode);
// set global objects for compiler web worker
window.clearTimeout = clearTimer;
window.console = console;
window.postMessage = postMessage;
window.setTimeout = setTimeout;
window.workerMain = workerMain;
window.close = workerClose;
window.TextDecoder = TextDecoder;
window.TextEncoder = TextEncoder;
// provide the "main" function that will be called by the privileged side when // provide the "main" function that will be called by the privileged side when
// lazy instantiating the compiler web worker // lazy instantiating the compiler web worker
window.compilerMain = function compilerMain(): void { window.compilerMain = function compilerMain(): void {
// workerMain should have already been called since a compiler is a worker. // workerMain should have already been called since a compiler is a worker.
window.onmessage = ({ data }: { data: CompilerLookup }): void => { window.onmessage = ({ data }: { data: CompilerReq }): void => {
const { specifier, referrer, cmdId } = data; const { rootNames, configPath, config } = data;
const host = new Host();
try { if (config && config.length) {
const result = compiler.compile(specifier, referrer); const ignoredOptions = host.configure(configPath!, config);
postMessage({ if (ignoredOptions) {
success: true, console.warn(
cmdId, yellow(`Unsupported compiler options in "${configPath}"\n`) +
data: result cyan(` The following options were ignored:\n`) +
}); ` ${ignoredOptions
} catch (e) { .map((value): string => bold(value))
postMessage({ .join(", ")}`
success: false, );
cmdId, }
data: JSON.parse(core.errorToJSON(e))
});
} }
const options = host.getCompilationSettings();
const program = ts.createProgram(rootNames, options, host);
const emitResult = program!.emit();
// TODO(ry) Print diagnostics in Rust.
// https://github.com/denoland/deno/pull/2310
const diagnostics = ts
.getPreEmitDiagnostics(program)
.concat(emitResult.diagnostics)
.filter(
({ code }): boolean => {
if (code === 2649) return false;
// TS2691: An import path cannot end with a '.ts' extension. Consider
// importing 'bad-module' instead.
if (code === 2691) return false;
// TS5009: Cannot find the common subdirectory path for the input files.
if (code === 5009) return false;
// TS5055: Cannot write file
// 'http://localhost:4545/tests/subdir/mt_application_x_javascript.j4.js'
// because it would overwrite input file.
if (code === 5055) return false;
// TypeScript is overly opinionated that only CommonJS modules kinds can
// support JSON imports. Allegedly this was fixed in
// Microsoft/TypeScript#26825 but that doesn't seem to be working here,
// so we will ignore complaints about this compiler setting.
if (code === 5070) return false;
return true;
}
);
if (diagnostics.length > 0) {
host._logDiagnostics(diagnostics);
// The above _logDiagnostics calls os.exit(). The return is here just for
// clarity.
return;
}
postMessage(emitResult);
// The compiler isolate exits after a single messsage.
workerClose();
}; };
}; };
const decoder = new TextDecoder();
// Perform the op to retrieve the compiler configuration if there was any
// provided on startup.
function getCompilerConfig(
compilerType: string
): { path: string; data: string } {
const builder = flatbuffers.createBuilder();
const compilerType_ = builder.createString(compilerType);
msg.CompilerConfig.startCompilerConfig(builder);
msg.CompilerConfig.addCompilerType(builder, compilerType_);
const inner = msg.CompilerConfig.endCompilerConfig(builder);
const baseRes = sendSync(builder, msg.Any.CompilerConfig, inner);
assert(baseRes != null);
assert(msg.Any.CompilerConfigRes === baseRes!.innerType());
const res = new msg.CompilerConfigRes();
assert(baseRes!.inner(res) != null);
// the privileged side does not normalize path separators in windows, so we
// will normalize them here
const path = res.path()!.replace(/\\/g, "/");
assert(path != null);
const dataArray = res.dataArray()!;
assert(dataArray != null);
const data = decoder.decode(dataArray);
return { path, data };
}
export default function denoMain(): void {
os.start("TS");
const { path, data } = getCompilerConfig("typescript");
if (data.length) {
const ignoredOptions = compiler.configure(path, data);
if (ignoredOptions) {
console.warn(
yellow(`Unsupported compiler options in "${path}"\n`) +
cyan(` The following options were ignored:\n`) +
` ${ignoredOptions.map((value): string => bold(value)).join(", ")}`
);
}
}
}

45
js/os.ts
View file

@ -3,7 +3,6 @@ import * as msg from "gen/cli/msg_generated";
import { core } from "./core"; import { core } from "./core";
import { handleAsyncMsgFromRust, sendSync } from "./dispatch"; import { handleAsyncMsgFromRust, sendSync } from "./dispatch";
import * as flatbuffers from "./flatbuffers"; import * as flatbuffers from "./flatbuffers";
import { TextDecoder } from "./text_encoding";
import { assert } from "./util"; import { assert } from "./util";
import * as util from "./util"; import * as util from "./util";
import { window } from "./window"; import { window } from "./window";
@ -24,13 +23,6 @@ function setGlobals(pid_: number, noColor_: boolean, execPath_: string): void {
execPath = execPath_; execPath = execPath_;
} }
interface ResponseModuleMetaData {
moduleName: string | undefined;
filename: string | undefined;
mediaType: msg.MediaType;
sourceCode: string | undefined;
}
/** Check if running in terminal. /** Check if running in terminal.
* *
* console.log(Deno.isTTY().stdout); * console.log(Deno.isTTY().stdout);
@ -54,43 +46,6 @@ export function exit(exitCode = 0): never {
return util.unreachable(); return util.unreachable();
} }
const decoder = new TextDecoder();
// @internal
export function fetchModuleMetaData(
specifier: string,
referrer: string
): ResponseModuleMetaData {
util.log("os.fetchModuleMetaData", { specifier, referrer });
// Send FetchModuleMetaData message
const builder = flatbuffers.createBuilder();
const specifier_ = builder.createString(specifier);
const referrer_ = builder.createString(referrer);
const inner = msg.FetchModuleMetaData.createFetchModuleMetaData(
builder,
specifier_,
referrer_
);
const baseRes = sendSync(builder, msg.Any.FetchModuleMetaData, inner);
assert(baseRes != null);
assert(
msg.Any.FetchModuleMetaDataRes === baseRes!.innerType(),
`base.innerType() unexpectedly is ${baseRes!.innerType()}`
);
const fetchModuleMetaDataRes = new msg.FetchModuleMetaDataRes();
assert(baseRes!.inner(fetchModuleMetaDataRes) != null);
const dataArray = fetchModuleMetaDataRes.dataArray();
const sourceCode = dataArray ? decoder.decode(dataArray) : undefined;
// flatbuffers returns `null` for an empty value, this does not fit well with
// idiomatic TypeScript under strict null checks, so converting to `undefined`
return {
moduleName: fetchModuleMetaDataRes.moduleName() || undefined,
filename: fetchModuleMetaDataRes.filename() || undefined,
mediaType: fetchModuleMetaDataRes.mediaType(),
sourceCode
};
}
function setEnv(key: string, value: string): void { function setEnv(key: string, value: string): void {
const builder = flatbuffers.createBuilder(); const builder = flatbuffers.createBuilder();
const key_ = builder.createString(key); const key_ = builder.createString(key);

4
rollup.config.js
View file

@ -231,10 +231,14 @@ export default function makeConfig(commandOptions) {
[typescriptPath]: [ [typescriptPath]: [
"convertCompilerOptionsFromJson", "convertCompilerOptionsFromJson",
"createLanguageService", "createLanguageService",
"createProgram",
"createSourceFile",
"getPreEmitDiagnostics",
"formatDiagnostics", "formatDiagnostics",
"formatDiagnosticsWithColorAndContext", "formatDiagnosticsWithColorAndContext",
"parseConfigFileTextToJson", "parseConfigFileTextToJson",
"version", "version",
"CompilerHost",
"Extension", "Extension",
"ModuleKind", "ModuleKind",
"ScriptKind", "ScriptKind",

3
tests/config.ts.out
View file

@ -1,7 +1,6 @@
Unsupported compiler options in "[WILDCARD]tests/config.tsconfig.json" [WILDCARD]Unsupported compiler options in "[WILDCARD]config.tsconfig.json"
The following options were ignored: The following options were ignored:
module, target module, target
[WILDCARD]tests/config.ts:3:5 - error TS2532: Object is possibly 'undefined'. [WILDCARD]tests/config.ts:3:5 - error TS2532: Object is possibly 'undefined'.
3 if (map.get("bar").foo) { 3 if (map.get("bar").foo) {

4
tests/error_004_missing_module.ts.out
View file

@ -3,9 +3,9 @@
at maybeError (js/errors.ts:[WILDCARD]) at maybeError (js/errors.ts:[WILDCARD])
at maybeThrowError (js/errors.ts:[WILDCARD]) at maybeThrowError (js/errors.ts:[WILDCARD])
at sendSync (js/dispatch.ts:[WILDCARD]) at sendSync (js/dispatch.ts:[WILDCARD])
at fetchModuleMetaData (js/os.ts:[WILDCARD]) at fetchModuleMetaData (js/compiler.ts:[WILDCARD])
at _resolveModule (js/compiler.ts:[WILDCARD]) at _resolveModule (js/compiler.ts:[WILDCARD])
at js/compiler.ts:[WILDCARD]
at resolveModuleNames (js/compiler.ts:[WILDCARD]) at resolveModuleNames (js/compiler.ts:[WILDCARD])
at compilerHost.resolveModuleNames ([WILDCARD]typescript.js:[WILDCARD])
at resolveModuleNamesWorker ([WILDCARD]typescript.js:[WILDCARD]) at resolveModuleNamesWorker ([WILDCARD]typescript.js:[WILDCARD])
at resolveModuleNamesReusingOldState ([WILDCARD]typescript.js:[WILDCARD]) at resolveModuleNamesReusingOldState ([WILDCARD]typescript.js:[WILDCARD])

5
tests/error_005_missing_dynamic_import.ts.out
View file

@ -3,9 +3,8 @@
at maybeError (js/errors.ts:[WILDCARD]) at maybeError (js/errors.ts:[WILDCARD])
at maybeThrowError (js/errors.ts:[WILDCARD]) at maybeThrowError (js/errors.ts:[WILDCARD])
at sendSync (js/dispatch.ts:[WILDCARD]) at sendSync (js/dispatch.ts:[WILDCARD])
at fetchModuleMetaData (js/os.ts:[WILDCARD]) at fetchModuleMetaData (js/compiler.ts:[WILDCARD])
at _resolveModule (js/compiler.ts:[WILDCARD]) at _resolveModule (js/compiler.ts:[WILDCARD])
at resolveModuleNames (js/compiler.ts:[WILDCARD]) at js/compiler.ts:[WILDCARD]
at compilerHost.resolveModuleNames ([WILDCARD])
at resolveModuleNamesWorker ([WILDCARD]) at resolveModuleNamesWorker ([WILDCARD])
at resolveModuleNamesReusingOldState ([WILDCARD]typescript.js:[WILDCARD]) at resolveModuleNamesReusingOldState ([WILDCARD]typescript.js:[WILDCARD])

5
tests/error_006_import_ext_failure.ts.out
View file

@ -3,9 +3,8 @@
at maybeError (js/errors.ts:[WILDCARD]) at maybeError (js/errors.ts:[WILDCARD])
at maybeThrowError (js/errors.ts:[WILDCARD]) at maybeThrowError (js/errors.ts:[WILDCARD])
at sendSync (js/dispatch.ts:[WILDCARD]) at sendSync (js/dispatch.ts:[WILDCARD])
at fetchModuleMetaData (js/os.ts:[WILDCARD]) at fetchModuleMetaData (js/compiler.ts:[WILDCARD])
at _resolveModule (js/compiler.ts:[WILDCARD]) at _resolveModule (js/compiler.ts:[WILDCARD])
at resolveModuleNames (js/compiler.ts:[WILDCARD]) at js/compiler.ts:[WILDCARD]
at compilerHost.resolveModuleNames ([WILDCARD])
at resolveModuleNamesWorker ([WILDCARD]) at resolveModuleNamesWorker ([WILDCARD])
at resolveModuleNamesReusingOldState ([WILDCARD]typescript.js:[WILDCARD]) at resolveModuleNamesReusingOldState ([WILDCARD]typescript.js:[WILDCARD])