First pass at HTTP imports

Implement --reload

Integrate hyper errors into DenoError

In collaboration with Tommy Savaria <tommy.savaria@protonmail.ch>

src/deno_dir.rs

@@ -1,6 +1,8 @@
 // Copyright 2018 the Deno authors. All rights reserved. MIT license.
 use errors::DenoError;
+use errors::DenoResult;
 use fs;
+use net;
 use sha1;
 use std;
 use std::fs::File;
@@ -25,12 +27,17 @@ pub struct DenoDir {
   // This is where we cache compilation outputs. Example:
   // /Users/rld/.deno/gen/f39a473452321cacd7c346a870efb0e3e1264b43.js
   pub deps: PathBuf,
+  // If remote resources should be reloaded.
+  reload: bool,
 }
 
 impl DenoDir {
   // Must be called before using any function from this module.
   // https://github.com/denoland/deno/blob/golang/deno_dir.go#L99-L111
-  pub fn new(custom_root: Option<&Path>) -> std::io::Result<DenoDir> {
+  pub fn new(
+    reload: bool,
+    custom_root: Option<&Path>,
+  ) -> std::io::Result<DenoDir> {
     // Only setup once.
     let home_dir = std::env::home_dir().expect("Could not get home directory.");
     let default = home_dir.join(".deno");
@@ -42,7 +49,12 @@ impl DenoDir {
     let gen = root.as_path().join("gen");
     let deps = root.as_path().join("deps");
 
-    let deno_dir = DenoDir { root, gen, deps };
+    let deno_dir = DenoDir {
+      root,
+      gen,
+      deps,
+      reload,
+    };
     fs::mkdir(deno_dir.gen.as_ref())?;
     fs::mkdir(deno_dir.deps.as_ref())?;
 
@@ -93,6 +105,50 @@ impl DenoDir {
     }
   }
 
+  // Prototype https://github.com/denoland/deno/blob/golang/deno_dir.go#L37-L73
+  fn fetch_remote_source(
+    self: &DenoDir,
+    module_name: &str,
+    filename: &str,
+  ) -> DenoResult<String> {
+    let p = Path::new(filename);
+
+    let src = if self.reload || !p.exists() {
+      println!("Downloading {}", module_name);
+      let source = net::fetch_sync_string(module_name)?;
+      match p.parent() {
+        Some(ref parent) => std::fs::create_dir_all(parent),
+        None => Ok(()),
+      }?;
+      fs::write_file_sync(&p, source.as_bytes())?;
+      source
+    } else {
+      let source = fs::read_file_sync_string(&p)?;
+      source
+    };
+    Ok(src)
+  }
+
+  // Prototype: https://github.com/denoland/deno/blob/golang/os.go#L122-L138
+  fn get_source_code(
+    self: &DenoDir,
+    module_name: &str,
+    filename: &str,
+  ) -> DenoResult<String> {
+    if is_remote(module_name) {
+      self.fetch_remote_source(module_name, filename)
+    } else if module_name.starts_with(ASSET_PREFIX) {
+      panic!("Asset resolution should be done in JS, not Rust.");
+    } else {
+      assert!(
+        module_name == filename,
+        "if a module isn't remote, it should have the same filename"
+      );
+      let src = fs::read_file_sync_string(Path::new(filename))?;
+      Ok(src)
+    }
+  }
+
   pub fn code_fetch(
     self: &DenoDir,
     module_specifier: &str,
@@ -106,7 +162,8 @@ impl DenoDir {
             module_name, module_specifier, containing_file, filename
         );
 
-    let out = get_source_code(module_name.as_str(), filename.as_str())
+    let out = self
+      .get_source_code(module_name.as_str(), filename.as_str())
       .and_then(|source_code| {
         Ok(CodeFetchOutput {
           module_name,
@@ -154,6 +211,9 @@ impl DenoDir {
     module_specifier: &str,
     containing_file: &str,
   ) -> Result<(String, String), url::ParseError> {
+    let module_name;
+    let filename;
+
     debug!(
       "resolve_module before module_specifier {} containing_file {}",
       module_specifier, containing_file
@@ -165,12 +225,11 @@ impl DenoDir {
 
     let j: Url =
       if containing_file == "." || Path::new(module_specifier).is_absolute() {
-        let r = Url::from_file_path(module_specifier);
+        if module_specifier.starts_with("http://") {
-        // TODO(ry) Properly handle error.
+          Url::parse(module_specifier)?
-        if r.is_err() {
+        } else {
-          error!("Url::from_file_path error {}", module_specifier);
+          Url::from_file_path(module_specifier).unwrap()
         }
-        r.unwrap()
       } else if containing_file.ends_with("/") {
         let r = Url::from_directory_path(&containing_file);
         // TODO(ry) Properly handle error.
@@ -189,27 +248,59 @@ impl DenoDir {
         base.join(module_specifier)?
       };
 
-    let mut p = j
+    match j.scheme() {
-      .to_file_path()
+      "file" => {
-      .unwrap()
+        let mut p = j
-      .into_os_string()
+          .to_file_path()
-      .into_string()
+          .unwrap()
-      .unwrap();
+          .into_os_string()
+          .into_string()
+          .unwrap();
 
-    if cfg!(target_os = "windows") {
+        if cfg!(target_os = "windows") {
-      // On windows, replace backward slashes to forward slashes.
+          // On windows, replace backward slashes to forward slashes.
-      // TODO(piscisaureus): This may not me be right, I just did it to make
+          // TODO(piscisaureus): This may not me be right, I just did it to make
-      // the tests pass.
+          // the tests pass.
-      p = p.replace("\\", "/");
+          p = p.replace("\\", "/");
+        }
+
+        module_name = p.to_string();
+        filename = p.to_string();
+      }
+      _ => {
+        module_name = module_specifier.to_string();
+        filename = get_cache_filename(self.deps.as_path(), j)
+          .to_str()
+          .unwrap()
+          .to_string();
+      }
     }
 
-    let module_name = p.to_string();
+    debug!("module_name: {}, filename: {}", module_name, filename);
-    let filename = p.to_string();
-
     Ok((module_name, filename))
   }
 }
 
+fn get_cache_filename(basedir: &Path, url: Url) -> PathBuf {
+  let mut out = basedir.to_path_buf();
+  out.push(url.host_str().unwrap());
+  for path_seg in url.path_segments().unwrap() {
+    out.push(path_seg);
+  }
+  out
+}
+
+#[test]
+fn test_get_cache_filename() {
+  let url = Url::parse("http://example.com:1234/path/to/file.ts").unwrap();
+  let basedir = Path::new("/cache/dir/");
+  let cache_file = get_cache_filename(&basedir, url);
+  assert_eq!(
+    cache_file,
+    Path::new("/cache/dir/example.com/path/to/file.ts")
+  );
+}
+
 #[derive(Debug)]
 pub struct CodeFetchOutput {
   pub module_name: String,
@@ -221,7 +312,8 @@ pub struct CodeFetchOutput {
 #[cfg(test)]
 pub fn test_setup() -> (TempDir, DenoDir) {
   let temp_dir = TempDir::new().expect("tempdir fail");
-  let deno_dir = DenoDir::new(Some(temp_dir.path())).expect("setup fail");
+  let deno_dir =
+    DenoDir::new(false, Some(temp_dir.path())).expect("setup fail");
   (temp_dir, deno_dir)
 }
 
@@ -395,24 +487,6 @@ fn test_resolve_module() {
 
 const ASSET_PREFIX: &str = "/$asset$/";
 
-fn is_remote(_module_name: &str) -> bool {
+fn is_remote(module_name: &str) -> bool {
-  false
+  module_name.starts_with("http")
-}
-
-fn get_source_code(
-  module_name: &str,
-  filename: &str,
-) -> std::io::Result<String> {
-  if is_remote(module_name) {
-    unimplemented!();
-  } else if module_name.starts_with(ASSET_PREFIX) {
-    assert!(false, "Asset resolution should be done in JS, not Rust.");
-    unimplemented!();
-  } else {
-    assert!(
-      module_name == filename,
-      "if a module isn't remote, it should have the same filename"
-    );
-    fs::read_file_sync_string(Path::new(filename))
-  }
 }

src/errors.rs

@@ -1,4 +1,5 @@
 // Copyright 2018 the Deno authors. All rights reserved. MIT license.
+use hyper;
 use msg_generated::deno as msg;
 use std;
 use std::fmt;
@@ -14,6 +15,14 @@ pub struct DenoError {
   repr: Repr,
 }
 
+#[derive(Debug)]
+enum Repr {
+  // Simple(ErrorKind),
+  IoErr(io::Error),
+  UrlErr(url::ParseError),
+  HyperErr(hyper::Error),
+}
+
 impl DenoError {
   pub fn kind(&self) -> ErrorKind {
     match self.repr {
@@ -59,22 +68,30 @@ impl DenoError {
           Overflow => ErrorKind::Overflow,
         }
       }
+      Repr::HyperErr(ref err) => {
+        // For some reason hyper::errors::Kind is private.
+        if err.is_parse() {
+          ErrorKind::HttpParse
+        } else if err.is_user() {
+          ErrorKind::HttpUser
+        } else if err.is_canceled() {
+          ErrorKind::HttpCanceled
+        } else if err.is_closed() {
+          ErrorKind::HttpClosed
+        } else {
+          ErrorKind::HttpOther
+        }
+      }
     }
   }
 }
 
-#[derive(Debug)]
-enum Repr {
-  // Simple(ErrorKind),
-  IoErr(io::Error),
-  UrlErr(url::ParseError),
-}
-
 impl fmt::Display for DenoError {
   fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
     match self.repr {
       Repr::IoErr(ref err) => err.fmt(f),
       Repr::UrlErr(ref err) => err.fmt(f),
+      Repr::HyperErr(ref err) => err.fmt(f),
       // Repr::Simple(..) => Ok(()),
     }
   }
@@ -85,6 +102,7 @@ impl std::error::Error for DenoError {
     match self.repr {
       Repr::IoErr(ref err) => err.description(),
       Repr::UrlErr(ref err) => err.description(),
+      Repr::HyperErr(ref err) => err.description(),
       // Repr::Simple(..) => "FIXME",
     }
   }
@@ -93,6 +111,7 @@ impl std::error::Error for DenoError {
     match self.repr {
       Repr::IoErr(ref err) => Some(err),
       Repr::UrlErr(ref err) => Some(err),
+      Repr::HyperErr(ref err) => Some(err),
       // Repr::Simple(..) => None,
     }
   }
@@ -115,3 +134,12 @@ impl From<url::ParseError> for DenoError {
     }
   }
 }
+
+impl From<hyper::Error> for DenoError {
+  #[inline]
+  fn from(err: hyper::Error) -> DenoError {
+    DenoError {
+      repr: Repr::HyperErr(err),
+    }
+  }
+}

src/main.rs

@@ -17,6 +17,7 @@ mod errors;
 mod flags;
 mod fs;
 pub mod handlers;
+mod net;
 mod version;
 
 use libc::c_void;
@@ -48,7 +49,7 @@ impl Deno {
 
     let mut deno_box = Box::new(Deno {
       ptr: 0 as *const binding::DenoC,
-      dir: deno_dir::DenoDir::new(None).unwrap(),
+      dir: deno_dir::DenoDir::new(flags.reload, None).unwrap(),
       rt: tokio::runtime::current_thread::Runtime::new().unwrap(),
       timers: HashMap::new(),
       argv: argv_rest,

src/msg.fbs

@@ -53,6 +53,14 @@ enum ErrorKind: byte {
   RelativeUrlWithCannotBeABaseBase,
   SetHostOnCannotBeABaseUrl,
   Overflow,
+
+  // hyper errors
+
+  HttpUser,
+  HttpClosed,
+  HttpCanceled,
+  HttpParse,
+  HttpOther,
 }
 
 table Base {

src/net.rs

@@ -0,0 +1,29 @@
+use errors::DenoResult;
+use hyper::rt::{Future, Stream};
+use hyper::{Client, Uri};
+use tokio::runtime::current_thread::Runtime;
+
+// The CodeFetch message is used to load HTTP javascript resources and expects a
+// synchronous response, this utility method supports that.
+pub fn fetch_sync_string(module_name: &str) -> DenoResult<String> {
+  let url = module_name.parse::<Uri>().unwrap();
+  let client = Client::new();
+
+  // TODO Use Deno's RT
+  let mut rt = Runtime::new().unwrap();
+
+  let body = rt.block_on(
+    client
+      .get(url)
+      .and_then(|response| response.into_body().concat2()),
+  )?;
+  Ok(String::from_utf8(body.to_vec()).unwrap())
+}
+
+#[test]
+fn test_fetch_sync_string() {
+  // Relies on external http server. See tools/http_server.py
+  let p = fetch_sync_string("http://localhost:4545/package.json").unwrap();
+  println!("package.json len {}", p.len());
+  assert!(p.len() > 1);
+}

tests/006_url_imports.ts

@@ -0,0 +1,3 @@
+import { printHello } from "http://localhost:4545/tests/subdir/print_hello.ts";
+printHello();
+console.log("success");

tests/006_url_imports.ts.out

@@ -0,0 +1,3 @@
+Downloading http://localhost:4545/tests/subdir/print_hello.ts
+Hello
+success

tools/check_output_test.py

@@ -26,7 +26,7 @@ def check_output_test(deno_exe_filename):
         out_abs = os.path.join(tests_path, out_filename)
         with open(out_abs, 'r') as f:
             expected_out = f.read()
-        cmd = [deno_exe_filename, script_abs]
+        cmd = [deno_exe_filename, script_abs, "--reload"]
         expected_code = parse_exit_code(script)
         print " ".join(cmd)
         actual_code = 0