Refactor DenoDir (#2636)

* rename `ModuleMetaData` to `SourceFile` and remove TS specific
  functionality

* add `TsCompiler` struct encapsulating processing of TypeScript files

* move `SourceMapGetter` trait implementation to `//cli/compiler.rs`

* add low-level `DiskCache` API for general purpose caches and use it in
  `DenoDir` and `TsCompiler` for filesystem access

* don't use hash-like filenames for compiled modules, instead use
  metadata file for storing compilation hash

* add `SourceFileCache` for in-process caching of loaded files for fast
  subsequent access

* define `SourceFileFetcher` trait encapsulating loading of local and
  remote files and implement it for `DenoDir`

* define `use_cache` and `no_fetch` flags on `DenoDir` instead of using
  in fetch methods

cli/compiler.rs

@@ -1,57 +1,80 @@
 // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
+use crate::deno_dir::DenoDir;
+use crate::deno_dir::SourceFile;
+use crate::deno_dir::SourceFileFetcher;
 use crate::diagnostics::Diagnostic;
+use crate::disk_cache::DiskCache;
 use crate::msg;
 use crate::resources;
+use crate::source_maps::SourceMapGetter;
 use crate::startup_data;
 use crate::state::*;
-use crate::tokio_util;
+use crate::version;
 use crate::worker::Worker;
 use deno::Buf;
 use deno::ErrBox;
 use deno::ModuleSpecifier;
+use futures::future::Either;
 use futures::Future;
 use futures::Stream;
+use ring;
+use std::collections::HashSet;
+use std::fmt::Write;
+use std::fs;
 use std::path::PathBuf;
 use std::str;
 use std::sync::atomic::Ordering;
+use std::sync::Mutex;
+use url::Url;
 
-// This corresponds to JS ModuleMetaData.
-// TODO Rename one or the other so they correspond.
-// TODO(bartlomieju): change `*_name` to `*_url` and use Url type
-#[derive(Debug, Clone)]
-pub struct ModuleMetaData {
-  pub module_name: String,
-  pub module_redirect_source_name: Option<String>, // source of redirect
-  pub filename: PathBuf,
-  pub media_type: msg::MediaType,
-  pub source_code: Vec<u8>,
-  pub maybe_output_code_filename: Option<PathBuf>,
-  pub maybe_output_code: Option<Vec<u8>>,
-  pub maybe_source_map_filename: Option<PathBuf>,
-  pub maybe_source_map: Option<Vec<u8>>,
+/// Optional tuple which represents the state of the compiler
+/// configuration where the first is canonical name for the configuration file
+/// and a vector of the bytes of the contents of the configuration file.
+type CompilerConfig = Option<(PathBuf, Vec<u8>)>;
+
+/// Information associated with compiled file in cache.
+/// Includes source code path and state hash.
+/// version_hash is used to validate versions of the file
+/// and could be used to remove stale file in cache.
+pub struct CompiledFileMetadata {
+  pub source_path: PathBuf,
+  pub version_hash: String,
 }
 
-impl ModuleMetaData {
-  pub fn has_output_code_and_source_map(&self) -> bool {
-    self.maybe_output_code.is_some() && self.maybe_source_map.is_some()
+static SOURCE_PATH: &'static str = "source_path";
+static VERSION_HASH: &'static str = "version_hash";
+
+impl CompiledFileMetadata {
+  pub fn from_json_string(metadata_string: String) -> Option<Self> {
+    // TODO: use serde for deserialization
+    let maybe_metadata_json: serde_json::Result<serde_json::Value> =
+      serde_json::from_str(&metadata_string);
+
+    if let Ok(metadata_json) = maybe_metadata_json {
+      let source_path = metadata_json[SOURCE_PATH].as_str().map(PathBuf::from);
+      let version_hash = metadata_json[VERSION_HASH].as_str().map(String::from);
+
+      if source_path.is_none() || version_hash.is_none() {
+        return None;
+      }
+
+      return Some(CompiledFileMetadata {
+        source_path: source_path.unwrap(),
+        version_hash: version_hash.unwrap(),
+      });
+    }
+
+    None
   }
 
-  pub fn js_source(&self) -> String {
-    if self.media_type == msg::MediaType::Json {
-      return format!(
-        "export default {};",
-        str::from_utf8(&self.source_code).unwrap()
-      );
-    }
-    match self.maybe_output_code {
-      None => str::from_utf8(&self.source_code).unwrap().to_string(),
-      Some(ref output_code) => str::from_utf8(output_code).unwrap().to_string(),
-    }
+  pub fn to_json_string(self: &Self) -> Result<String, serde_json::Error> {
+    let mut value_map = serde_json::map::Map::new();
+
+    value_map.insert(SOURCE_PATH.to_owned(), json!(&self.source_path));
+    value_map.insert(VERSION_HASH.to_string(), json!(&self.version_hash));
+    serde_json::to_string(&value_map)
   }
 }
-
-type CompilerConfig = Option<(String, Vec<u8>)>;
-
 /// Creates the JSON message send to compiler.ts's onmessage.
 fn req(
   root_names: Vec<String>,
@@ -74,229 +97,565 @@ fn req(
   j.to_string().into_boxed_str().into_boxed_bytes()
 }
 
-/// Returns an optional tuple which represents the state of the compiler
-/// configuration where the first is canonical name for the configuration file
-/// and a vector of the bytes of the contents of the configuration file.
-pub fn get_compiler_config(
-  parent_state: &ThreadSafeState,
-  _compiler_type: &str,
-) -> CompilerConfig {
-  // The compiler type is being passed to make it easier to implement custom
-  // compilers in the future.
-  match (&parent_state.config_path, &parent_state.config) {
-    (Some(config_path), Some(config)) => {
-      Some((config_path.to_string(), config.to_vec()))
+fn gen_hash(v: Vec<&[u8]>) -> String {
+  let mut ctx = ring::digest::Context::new(&ring::digest::SHA1);
+  for src in v.iter() {
+    ctx.update(src);
+  }
+  let digest = ctx.finish();
+  let mut out = String::new();
+  // TODO There must be a better way to do this...
+  for byte in digest.as_ref() {
+    write!(&mut out, "{:02x}", byte).unwrap();
+  }
+  out
+}
+
+/// Emit a SHA1 hash based on source code, deno version and TS config.
+/// Used to check if a recompilation for source code is needed.
+pub fn source_code_version_hash(
+  source_code: &[u8],
+  version: &str,
+  config_hash: &[u8],
+) -> String {
+  gen_hash(vec![source_code, version.as_bytes(), config_hash])
+}
+
+fn load_config_file(
+  config_path: Option<String>,
+) -> (Option<PathBuf>, Option<Vec<u8>>) {
+  // take the passed flag and resolve the file name relative to the cwd
+  let config_file = match &config_path {
+    Some(config_file_name) => {
+      debug!("Compiler config file: {}", config_file_name);
+      let cwd = std::env::current_dir().unwrap();
+      Some(cwd.join(config_file_name))
     }
     _ => None,
+  };
+
+  // Convert the PathBuf to a canonicalized string.  This is needed by the
+  // compiler to properly deal with the configuration.
+  let config_path = match &config_file {
+    Some(config_file) => Some(config_file.canonicalize().unwrap().to_owned()),
+    _ => None,
+  };
+
+  // Load the contents of the configuration file
+  let config = match &config_file {
+    Some(config_file) => {
+      debug!("Attempt to load config: {}", config_file.to_str().unwrap());
+      match fs::read(&config_file) {
+        Ok(config_data) => Some(config_data.to_owned()),
+        _ => panic!(
+          "Error retrieving compiler config file at \"{}\"",
+          config_file.to_str().unwrap()
+        ),
+      }
+    }
+    _ => None,
+  };
+
+  (config_path, config)
+}
+
+pub struct TsCompiler {
+  pub deno_dir: DenoDir,
+  pub config: CompilerConfig,
+  pub config_hash: Vec<u8>,
+  pub disk_cache: DiskCache,
+  /// Set of all URLs that have been compiled. This prevents double
+  /// compilation of module.
+  pub compiled: Mutex<HashSet<Url>>,
+  /// This setting is controlled by `--reload` flag. Unless the flag
+  /// is provided disk cache is used.
+  pub use_disk_cache: bool,
+}
+
+impl TsCompiler {
+  pub fn new(
+    deno_dir: DenoDir,
+    use_disk_cache: bool,
+    config_path: Option<String>,
+  ) -> Self {
+    let compiler_config = match load_config_file(config_path) {
+      (Some(config_path), Some(config)) => Some((config_path, config.to_vec())),
+      _ => None,
+    };
+
+    let config_bytes = match &compiler_config {
+      Some((_, config)) => config.clone(),
+      _ => b"".to_vec(),
+    };
+
+    Self {
+      disk_cache: deno_dir.clone().gen_cache,
+      deno_dir,
+      config: compiler_config,
+      config_hash: config_bytes,
+      compiled: Mutex::new(HashSet::new()),
+      use_disk_cache,
+    }
+  }
+
+  /// Create a new V8 worker with snapshot of TS compiler and setup compiler's runtime.
+  fn setup_worker(state: ThreadSafeState) -> Worker {
+    // Count how many times we start the compiler worker.
+    state.metrics.compiler_starts.fetch_add(1, Ordering::SeqCst);
+
+    let mut worker = Worker::new(
+      "TS".to_string(),
+      startup_data::compiler_isolate_init(),
+      // TODO(ry) Maybe we should use a separate state for the compiler.
+      // as was done previously.
+      state.clone(),
+    );
+    worker.execute("denoMain()").unwrap();
+    worker.execute("workerMain()").unwrap();
+    worker.execute("compilerMain()").unwrap();
+    worker
+  }
+
+  pub fn bundle_async(
+    self: &Self,
+    state: ThreadSafeState,
+    module_name: String,
+    out_file: String,
+  ) -> impl Future<Item = (), Error = ErrBox> {
+    debug!(
+      "Invoking the compiler to bundle. module_name: {}",
+      module_name
+    );
+
+    let root_names = vec![module_name.clone()];
+    let req_msg = req(root_names, self.config.clone(), Some(out_file));
+
+    let worker = TsCompiler::setup_worker(state.clone());
+    let resource = worker.state.resource.clone();
+    let compiler_rid = resource.rid;
+    let first_msg_fut =
+      resources::post_message_to_worker(compiler_rid, req_msg)
+        .then(move |_| worker)
+        .then(move |result| {
+          if let Err(err) = result {
+            // TODO(ry) Need to forward the error instead of exiting.
+            eprintln!("{}", err.to_string());
+            std::process::exit(1);
+          }
+          debug!("Sent message to worker");
+          let stream_future =
+            resources::get_message_stream_from_worker(compiler_rid)
+              .into_future();
+          stream_future.map(|(f, _rest)| f).map_err(|(f, _rest)| f)
+        });
+
+    first_msg_fut.map_err(|_| panic!("not handled")).and_then(
+      move |maybe_msg: Option<Buf>| {
+        debug!("Received message from worker");
+
+        if let Some(msg) = maybe_msg {
+          let json_str = std::str::from_utf8(&msg).unwrap();
+          debug!("Message: {}", json_str);
+          if let Some(diagnostics) = Diagnostic::from_emit_result(json_str) {
+            return Err(ErrBox::from(diagnostics));
+          }
+        }
+
+        Ok(())
+      },
+    )
+  }
+
+  /// Mark given module URL as compiled to avoid multiple compilations of same module
+  /// in single run.
+  fn mark_compiled(&self, url: &Url) {
+    let mut c = self.compiled.lock().unwrap();
+    c.insert(url.clone());
+  }
+
+  /// Check if given module URL has already been compiled and can be fetched directly from disk.
+  fn has_compiled(&self, url: &Url) -> bool {
+    let c = self.compiled.lock().unwrap();
+    c.contains(url)
+  }
+
+  /// Asynchronously compile module and all it's dependencies.
+  ///
+  /// This method compiled every module at most once.
+  ///
+  /// If `--reload` flag was provided then compiler will not on-disk cache and force recompilation.
+  ///
+  /// If compilation is required then new V8 worker is spawned with fresh TS compiler.
+  pub fn compile_async(
+    self: &Self,
+    state: ThreadSafeState,
+    source_file: &SourceFile,
+  ) -> impl Future<Item = SourceFile, Error = ErrBox> {
+    // TODO: maybe fetching of original SourceFile should be done here?
+
+    if source_file.media_type != msg::MediaType::TypeScript {
+      return Either::A(futures::future::ok(source_file.clone()));
+    }
+
+    if self.has_compiled(&source_file.url) {
+      match self.get_compiled_source_file(&source_file) {
+        Ok(compiled_module) => {
+          return Either::A(futures::future::ok(compiled_module));
+        }
+        Err(err) => {
+          return Either::A(futures::future::err(err));
+        }
+      }
+    }
+
+    if self.use_disk_cache {
+      // Try to load cached version:
+      // 1. check if there's 'meta' file
+      if let Some(metadata) = self.get_metadata(&source_file.url) {
+        // 2. compare version hashes
+        // TODO: it would probably be good idea to make it method implemented on SourceFile
+        let version_hash_to_validate = source_code_version_hash(
+          &source_file.source_code,
+          version::DENO,
+          &self.config_hash,
+        );
+
+        if metadata.version_hash == version_hash_to_validate {
+          debug!("load_cache metadata version hash match");
+          if let Ok(compiled_module) =
+            self.get_compiled_source_file(&source_file)
+          {
+            debug!(
+              "found cached compiled module: {:?}",
+              compiled_module.clone().filename
+            );
+            // TODO: store in in-process cache for subsequent access
+            return Either::A(futures::future::ok(compiled_module));
+          }
+        }
+      }
+    }
+
+    let source_file_ = source_file.clone();
+
+    debug!(">>>>> compile_sync START");
+    let module_url = source_file.url.clone();
+
+    debug!(
+      "Running rust part of compile_sync, module specifier: {}",
+      &source_file.url
+    );
+
+    let root_names = vec![module_url.to_string()];
+    let req_msg = req(root_names, self.config.clone(), None);
+
+    let worker = TsCompiler::setup_worker(state.clone());
+    let compiling_job = state.progress.add("Compile", &module_url.to_string());
+    let state_ = state.clone();
+
+    let resource = worker.state.resource.clone();
+    let compiler_rid = resource.rid;
+    let first_msg_fut =
+      resources::post_message_to_worker(compiler_rid, req_msg)
+        .then(move |_| worker)
+        .then(move |result| {
+          if let Err(err) = result {
+            // TODO(ry) Need to forward the error instead of exiting.
+            eprintln!("{}", err.to_string());
+            std::process::exit(1);
+          }
+          debug!("Sent message to worker");
+          let stream_future =
+            resources::get_message_stream_from_worker(compiler_rid)
+              .into_future();
+          stream_future.map(|(f, _rest)| f).map_err(|(f, _rest)| f)
+        });
+
+    let fut = first_msg_fut
+      .map_err(|_| panic!("not handled"))
+      .and_then(move |maybe_msg: Option<Buf>| {
+        debug!("Received message from worker");
+
+        if let Some(msg) = maybe_msg {
+          let json_str = std::str::from_utf8(&msg).unwrap();
+          debug!("Message: {}", json_str);
+          if let Some(diagnostics) = Diagnostic::from_emit_result(json_str) {
+            return Err(ErrBox::from(diagnostics));
+          }
+        }
+
+        Ok(())
+      }).and_then(move |_| {
+        // if we are this far it means compilation was successful and we can
+        // load compiled filed from disk
+        // TODO: can this be somehow called using `self.`?
+        state_
+          .ts_compiler
+          .get_compiled_source_file(&source_file_)
+          .map_err(|e| {
+            // TODO: this situation shouldn't happen
+            panic!("Expected to find compiled file: {}", e)
+          })
+      }).and_then(move |source_file_after_compile| {
+        // Explicit drop to keep reference alive until future completes.
+        drop(compiling_job);
+
+        Ok(source_file_after_compile)
+      }).then(move |r| {
+        debug!(">>>>> compile_sync END");
+        // TODO(ry) do this in worker's destructor.
+        // resource.close();
+        r
+      });
+
+    Either::B(fut)
+  }
+
+  /// Get associated `CompiledFileMetadata` for given module if it exists.
+  pub fn get_metadata(self: &Self, url: &Url) -> Option<CompiledFileMetadata> {
+    // Try to load cached version:
+    // 1. check if there's 'meta' file
+    let cache_key = self
+      .disk_cache
+      .get_cache_filename_with_extension(url, "meta");
+    if let Ok(metadata_bytes) = self.disk_cache.get(&cache_key) {
+      if let Ok(metadata) = std::str::from_utf8(&metadata_bytes) {
+        if let Some(read_metadata) =
+          CompiledFileMetadata::from_json_string(metadata.to_string())
+        {
+          return Some(read_metadata);
+        }
+      }
+    }
+
+    None
+  }
+
+  /// Return compiled JS file for given TS module.
+  // TODO: ideally we shouldn't construct SourceFile by hand, but it should be delegated to
+  // SourceFileFetcher
+  pub fn get_compiled_source_file(
+    self: &Self,
+    source_file: &SourceFile,
+  ) -> Result<SourceFile, ErrBox> {
+    let cache_key = self
+      .disk_cache
+      .get_cache_filename_with_extension(&source_file.url, "js");
+    let compiled_code = self.disk_cache.get(&cache_key)?;
+    let compiled_code_filename = self.disk_cache.location.join(cache_key);
+    debug!("compiled filename: {:?}", compiled_code_filename);
+
+    let compiled_module = SourceFile {
+      url: source_file.url.clone(),
+      redirect_source_url: None,
+      filename: compiled_code_filename,
+      media_type: msg::MediaType::JavaScript,
+      source_code: compiled_code,
+    };
+
+    Ok(compiled_module)
+  }
+
+  /// Save compiled JS file for given TS module to on-disk cache.
+  ///
+  /// Along compiled file a special metadata file is saved as well containing
+  /// hash that can be validated to avoid unnecessary recompilation.
+  fn cache_compiled_file(
+    self: &Self,
+    module_specifier: &ModuleSpecifier,
+    contents: &str,
+  ) -> std::io::Result<()> {
+    let js_key = self
+      .disk_cache
+      .get_cache_filename_with_extension(module_specifier.as_url(), "js");
+    self
+      .disk_cache
+      .set(&js_key, contents.as_bytes())
+      .and_then(|_| {
+        self.mark_compiled(module_specifier.as_url());
+
+        let source_file = self
+          .deno_dir
+          .fetch_source_file(&module_specifier)
+          .expect("Source file not found");
+
+        let version_hash = source_code_version_hash(
+          &source_file.source_code,
+          version::DENO,
+          &self.config_hash,
+        );
+
+        let compiled_file_metadata = CompiledFileMetadata {
+          source_path: source_file.filename.to_owned(),
+          version_hash,
+        };
+        let meta_key = self
+          .disk_cache
+          .get_cache_filename_with_extension(module_specifier.as_url(), "meta");
+        self.disk_cache.set(
+          &meta_key,
+          compiled_file_metadata.to_json_string()?.as_bytes(),
+        )
+      })
+  }
+
+  /// Return associated source map file for given TS module.
+  // TODO: ideally we shouldn't construct SourceFile by hand, but it should be delegated to
+  // SourceFileFetcher
+  pub fn get_source_map_file(
+    self: &Self,
+    module_specifier: &ModuleSpecifier,
+  ) -> Result<SourceFile, ErrBox> {
+    let cache_key = self
+      .disk_cache
+      .get_cache_filename_with_extension(module_specifier.as_url(), "js.map");
+    let source_code = self.disk_cache.get(&cache_key)?;
+    let source_map_filename = self.disk_cache.location.join(cache_key);
+    debug!("source map filename: {:?}", source_map_filename);
+
+    let source_map_file = SourceFile {
+      url: module_specifier.as_url().to_owned(),
+      redirect_source_url: None,
+      filename: source_map_filename,
+      media_type: msg::MediaType::JavaScript,
+      source_code,
+    };
+
+    Ok(source_map_file)
+  }
+
+  /// Save source map file for given TS module to on-disk cache.
+  fn cache_source_map(
+    self: &Self,
+    module_specifier: &ModuleSpecifier,
+    contents: &str,
+  ) -> std::io::Result<()> {
+    let source_map_key = self
+      .disk_cache
+      .get_cache_filename_with_extension(module_specifier.as_url(), "js.map");
+    self.disk_cache.set(&source_map_key, contents.as_bytes())
+  }
+
+  /// This method is called by TS compiler via an "op".
+  pub fn cache_compiler_output(
+    self: &Self,
+    module_specifier: &ModuleSpecifier,
+    extension: &str,
+    contents: &str,
+  ) -> std::io::Result<()> {
+    match extension {
+      ".map" => self.cache_source_map(module_specifier, contents),
+      ".js" => self.cache_compiled_file(module_specifier, contents),
+      _ => unreachable!(),
+    }
   }
 }
 
-pub fn bundle_async(
-  state: ThreadSafeState,
-  module_name: String,
-  out_file: String,
-) -> impl Future<Item = (), Error = ErrBox> {
-  debug!(
-    "Invoking the compiler to bundle. module_name: {}",
-    module_name
-  );
+impl SourceMapGetter for TsCompiler {
+  fn get_source_map(&self, script_name: &str) -> Option<Vec<u8>> {
+    self
+      .try_to_resolve_and_get_source_map(script_name)
+      .and_then(|out| Some(out.source_code))
+  }
 
-  let root_names = vec![module_name.clone()];
-  let compiler_config = get_compiler_config(&state, "typescript");
-  let req_msg = req(root_names, compiler_config, Some(out_file));
-
-  // Count how many times we start the compiler worker.
-  state.metrics.compiler_starts.fetch_add(1, Ordering::SeqCst);
-
-  let mut worker = Worker::new(
-    "TS".to_string(),
-    startup_data::compiler_isolate_init(),
-    // TODO(ry) Maybe we should use a separate state for the compiler.
-    // as was done previously.
-    state.clone(),
-  );
-  worker.execute("denoMain()").unwrap();
-  worker.execute("workerMain()").unwrap();
-  worker.execute("compilerMain()").unwrap();
-
-  let resource = worker.state.resource.clone();
-  let compiler_rid = resource.rid;
-  let first_msg_fut = resources::post_message_to_worker(compiler_rid, req_msg)
-    .then(move |_| worker)
-    .then(move |result| {
-      if let Err(err) = result {
-        // TODO(ry) Need to forward the error instead of exiting.
-        eprintln!("{}", err.to_string());
-        std::process::exit(1);
-      }
-      debug!("Sent message to worker");
-      let stream_future =
-        resources::get_message_stream_from_worker(compiler_rid).into_future();
-      stream_future.map(|(f, _rest)| f).map_err(|(f, _rest)| f)
-    });
-
-  first_msg_fut.map_err(|_| panic!("not handled")).and_then(
-    move |maybe_msg: Option<Buf>| {
-      debug!("Received message from worker");
-
-      if let Some(msg) = maybe_msg {
-        let json_str = std::str::from_utf8(&msg).unwrap();
-        debug!("Message: {}", json_str);
-        if let Some(diagnostics) = Diagnostic::from_emit_result(json_str) {
-          return Err(ErrBox::from(diagnostics));
-        }
-      }
-
-      Ok(())
-    },
-  )
-}
-
-pub fn compile_async(
-  state: ThreadSafeState,
-  module_meta_data: &ModuleMetaData,
-) -> impl Future<Item = ModuleMetaData, Error = ErrBox> {
-  let module_name = module_meta_data.module_name.clone();
-
-  debug!(
-    "Running rust part of compile_sync. module_name: {}",
-    &module_name
-  );
-
-  let root_names = vec![module_name.clone()];
-  let compiler_config = get_compiler_config(&state, "typescript");
-  let req_msg = req(root_names, compiler_config, None);
-
-  // Count how many times we start the compiler worker.
-  state.metrics.compiler_starts.fetch_add(1, Ordering::SeqCst);
-
-  let mut worker = Worker::new(
-    "TS".to_string(),
-    startup_data::compiler_isolate_init(),
-    // TODO(ry) Maybe we should use a separate state for the compiler.
-    // as was done previously.
-    state.clone(),
-  );
-  worker.execute("denoMain()").unwrap();
-  worker.execute("workerMain()").unwrap();
-  worker.execute("compilerMain()").unwrap();
-
-  let compiling_job = state.progress.add("Compile", &module_name);
-
-  let resource = worker.state.resource.clone();
-  let compiler_rid = resource.rid;
-  let first_msg_fut = resources::post_message_to_worker(compiler_rid, req_msg)
-    .then(move |_| worker)
-    .then(move |result| {
-      if let Err(err) = result {
-        // TODO(ry) Need to forward the error instead of exiting.
-        eprintln!("{}", err.to_string());
-        std::process::exit(1);
-      }
-      debug!("Sent message to worker");
-      let stream_future =
-        resources::get_message_stream_from_worker(compiler_rid).into_future();
-      stream_future.map(|(f, _rest)| f).map_err(|(f, _rest)| f)
-    });
-
-  first_msg_fut
-    .map_err(|_| panic!("not handled"))
-    .and_then(move |maybe_msg: Option<Buf>| {
-      debug!("Received message from worker");
-
-      // TODO: here TS compiler emitted the files to disc and we should signal ModuleMetaData
-      //  cache that source code is available
-      if let Some(msg) = maybe_msg {
-        let json_str = std::str::from_utf8(&msg).unwrap();
-        debug!("Message: {}", json_str);
-        if let Some(diagnostics) = Diagnostic::from_emit_result(json_str) {
-          return Err(ErrBox::from(diagnostics));
-        }
-      }
-
-      Ok(())
-    }).and_then(move |_| {
-      let module_specifier = ModuleSpecifier::resolve_url(&module_name)
-        .expect("Should be valid module specifier");
-      state.dir.fetch_module_meta_data_async(
-        &module_specifier,
-        true,
-        true,
-      ).map_err(|e| {
-        // TODO(95th) Instead of panicking, We could translate this error to Diagnostic.
-        panic!("{}", e)
+  fn get_source_line(&self, script_name: &str, line: usize) -> Option<String> {
+    self
+      .try_resolve_and_get_source_file(script_name)
+      .and_then(|out| {
+        str::from_utf8(&out.source_code).ok().and_then(|v| {
+          let lines: Vec<&str> = v.lines().collect();
+          assert!(lines.len() > line);
+          Some(lines[line].to_string())
+        })
       })
-    }).and_then(move |module_meta_data_after_compile| {
-      // Explicit drop to keep reference alive until future completes.
-      drop(compiling_job);
-
-      Ok(module_meta_data_after_compile)
-    }).then(move |r| {
-      // TODO(ry) do this in worker's destructor.
-      // resource.close();
-      r
-    })
+  }
 }
 
-pub fn compile_sync(
-  state: ThreadSafeState,
-  module_meta_data: &ModuleMetaData,
-) -> Result<ModuleMetaData, ErrBox> {
-  tokio_util::block_on(compile_async(state, module_meta_data))
+// `SourceMapGetter` related methods
+impl TsCompiler {
+  fn try_to_resolve(self: &Self, script_name: &str) -> Option<ModuleSpecifier> {
+    // if `script_name` can't be resolved to ModuleSpecifier it's probably internal
+    // script (like `gen/cli/bundle/compiler.js`) so we won't be
+    // able to get source for it anyway
+    ModuleSpecifier::resolve_url(script_name).ok()
+  }
+
+  fn try_resolve_and_get_source_file(
+    &self,
+    script_name: &str,
+  ) -> Option<SourceFile> {
+    if let Some(module_specifier) = self.try_to_resolve(script_name) {
+      return match self.deno_dir.fetch_source_file(&module_specifier) {
+        Ok(out) => Some(out),
+        Err(_) => None,
+      };
+    }
+
+    None
+  }
+
+  fn try_to_resolve_and_get_source_map(
+    &self,
+    script_name: &str,
+  ) -> Option<SourceFile> {
+    if let Some(module_specifier) = self.try_to_resolve(script_name) {
+      return match self.get_source_map_file(&module_specifier) {
+        Ok(out) => Some(out),
+        Err(_) => None,
+      };
+    }
+
+    None
+  }
 }
 
 #[cfg(test)]
 mod tests {
   use super::*;
+  use crate::tokio_util;
+  use deno::ModuleSpecifier;
+  use std::path::PathBuf;
+
+  impl TsCompiler {
+    fn compile_sync(
+      self: &Self,
+      state: ThreadSafeState,
+      source_file: &SourceFile,
+    ) -> Result<SourceFile, ErrBox> {
+      tokio_util::block_on(self.compile_async(state, source_file))
+    }
+  }
 
   #[test]
   fn test_compile_sync() {
     tokio_util::init(|| {
-      let specifier = "./tests/002_hello.ts";
-      use deno::ModuleSpecifier;
-      let module_name = ModuleSpecifier::resolve_url_or_path(specifier)
-        .unwrap()
-        .to_string();
+      let specifier =
+        ModuleSpecifier::resolve_url_or_path("./tests/002_hello.ts").unwrap();
 
-      let mut out = ModuleMetaData {
-        module_name,
-        module_redirect_source_name: None,
+      let mut out = SourceFile {
+        url: specifier.as_url().clone(),
+        redirect_source_url: None,
         filename: PathBuf::from("/tests/002_hello.ts"),
         media_type: msg::MediaType::TypeScript,
         source_code: include_bytes!("../tests/002_hello.ts").to_vec(),
-        maybe_output_code_filename: None,
-        maybe_output_code: None,
-        maybe_source_map_filename: None,
-        maybe_source_map: None,
       };
 
-      out = compile_sync(
-        ThreadSafeState::mock(vec![
-          String::from("./deno"),
-          String::from("hello.js"),
-        ]),
-        &out,
-      ).unwrap();
+      let mock_state = ThreadSafeState::mock(vec![
+        String::from("./deno"),
+        String::from("hello.js"),
+      ]);
+      out = mock_state
+        .ts_compiler
+        .compile_sync(mock_state.clone(), &out)
+        .unwrap();
       assert!(
         out
-          .maybe_output_code
-          .unwrap()
+          .source_code
           .starts_with("console.log(\"Hello World\");".as_bytes())
       );
     })
   }
 
-  #[test]
-  fn test_get_compiler_config_no_flag() {
-    let compiler_type = "typescript";
-    let state = ThreadSafeState::mock(vec![
-      String::from("./deno"),
-      String::from("hello.js"),
-    ]);
-    let out = get_compiler_config(&state, compiler_type);
-    assert_eq!(out, None);
-  }
-
   #[test]
   fn test_bundle_async() {
     let specifier = "./tests/002_hello.ts";
@@ -310,8 +669,34 @@ mod tests {
       String::from("./tests/002_hello.ts"),
       String::from("$deno$/bundle.js"),
     ]);
-    let out =
-      bundle_async(state, module_name, String::from("$deno$/bundle.js"));
+    let out = state.ts_compiler.bundle_async(
+      state.clone(),
+      module_name,
+      String::from("$deno$/bundle.js"),
+    );
     assert!(tokio_util::block_on(out).is_ok());
   }
+
+  #[test]
+  fn test_source_code_version_hash() {
+    assert_eq!(
+      "08574f9cdeb94fd3fb9cdc7a20d086daeeb42bca",
+      source_code_version_hash(b"1+2", "0.4.0", b"{}")
+    );
+    // Different source_code should result in different hash.
+    assert_eq!(
+      "d8abe2ead44c3ff8650a2855bf1b18e559addd06",
+      source_code_version_hash(b"1", "0.4.0", b"{}")
+    );
+    // Different version should result in different hash.
+    assert_eq!(
+      "d6feffc5024d765d22c94977b4fe5975b59d6367",
+      source_code_version_hash(b"1", "0.1.0", b"{}")
+    );
+    // Different config should result in different hash.
+    assert_eq!(
+      "3b35db249b26a27decd68686f073a58266b2aec2",
+      source_code_version_hash(b"1", "0.4.0", b"{\"compilerOptions\": {}}")
+    );
+  }
 }