chore: port http_server.py to rust (#6364)

Cargo.lock

@@ -121,6 +121,15 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d"
 
+[[package]]
+name = "base64"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e"
+dependencies = [
+ "byteorder",
+]
+
 [[package]]
 name = "base64"
 version = "0.11.0"
@@ -408,7 +417,7 @@ dependencies = [
  "termcolor",
  "test_util",
  "tokio",
- "tokio-rustls",
+ "tokio-rustls 0.13.1",
  "tokio-tungstenite",
  "url",
  "utime",
@@ -941,9 +950,9 @@ dependencies = [
  "futures-util",
  "hyper",
  "log 0.4.8",
- "rustls",
+ "rustls 0.17.0",
  "tokio",
- "tokio-rustls",
+ "tokio-rustls 0.13.1",
  "webpki",
 ]
 
@@ -1815,11 +1824,11 @@ dependencies = [
  "mime_guess 2.0.3",
  "percent-encoding",
  "pin-project-lite",
- "rustls",
+ "rustls 0.17.0",
  "serde",
  "serde_urlencoded",
  "tokio",
- "tokio-rustls",
+ "tokio-rustls 0.13.1",
  "url",
  "wasm-bindgen",
  "wasm-bindgen-futures",
@@ -1852,6 +1861,19 @@ dependencies = [
  "semver",
 ]
 
+[[package]]
+name = "rustls"
+version = "0.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b25a18b1bf7387f0145e7f8324e700805aade3842dd3db2e74e4cdeb4677c09e"
+dependencies = [
+ "base64 0.10.1",
+ "log 0.4.8",
+ "ring",
+ "sct",
+ "webpki",
+]
+
 [[package]]
 name = "rustls"
 version = "0.17.0"
@@ -2349,10 +2371,14 @@ dependencies = [
 name = "test_util"
 version = "0.1.0"
 dependencies = [
+ "bytes 0.5.5",
+ "futures 0.3.5",
  "lazy_static",
  "os_pipe",
  "regex",
  "tempfile",
+ "tokio",
+ "warp",
 ]
 
 [[package]]
@@ -2455,6 +2481,18 @@ dependencies = [
  "syn 1.0.33",
 ]
 
+[[package]]
+name = "tokio-rustls"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3068d891551949b37681724d6b73666787cc63fa8e255c812a41d2513aff9775"
+dependencies = [
+ "futures-core",
+ "rustls 0.16.0",
+ "tokio",
+ "webpki",
+]
+
 [[package]]
 name = "tokio-rustls"
 version = "0.13.1"
@@ -2462,7 +2500,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "15cb62a0d2770787abc96e99c1cd98fcf17f94959f3af63ca85bdfb203f051b4"
 dependencies = [
  "futures-core",
- "rustls",
+ "rustls 0.17.0",
  "tokio",
  "webpki",
 ]
@@ -2721,6 +2759,7 @@ dependencies = [
  "serde_json",
  "serde_urlencoded",
  "tokio",
+ "tokio-rustls 0.12.3",
  "tokio-tungstenite",
  "tower-service",
  "urlencoding",

cli/http_util.rs

@@ -249,7 +249,7 @@ mod tests {
   #[tokio::test]
   async fn test_fetch_string() {
     let http_server_guard = test_util::http_server();
-    // Relies on external http server. See tools/http_server.py
+    // Relies on external http server. See target/debug/test_server
     let url =
       Url::parse("http://127.0.0.1:4545/cli/tests/fixture.json").unwrap();
     let client = create_http_client(None).unwrap();
@@ -268,7 +268,7 @@ mod tests {
   #[tokio::test]
   async fn test_fetch_gzip() {
     let http_server_guard = test_util::http_server();
-    // Relies on external http server. See tools/http_server.py
+    // Relies on external http server. See target/debug/test_server
     let url = Url::parse(
       "http://127.0.0.1:4545/cli/tests/053_import_compression/gziped",
     )
@@ -317,7 +317,7 @@ mod tests {
   #[tokio::test]
   async fn test_fetch_brotli() {
     let http_server_guard = test_util::http_server();
-    // Relies on external http server. See tools/http_server.py
+    // Relies on external http server. See target/debug/test_server
     let url = Url::parse(
       "http://127.0.0.1:4545/cli/tests/053_import_compression/brotli",
     )
@@ -342,7 +342,7 @@ mod tests {
   #[tokio::test]
   async fn test_fetch_once_with_redirect() {
     let http_server_guard = test_util::http_server();
-    // Relies on external http server. See tools/http_server.py
+    // Relies on external http server. See target/debug/test_server
     let url =
       Url::parse("http://127.0.0.1:4546/cli/tests/fixture.json").unwrap();
     // Dns resolver substitutes `127.0.0.1` with `localhost`
@@ -399,7 +399,7 @@ mod tests {
   #[tokio::test]
   async fn test_fetch_with_cafile_string() {
     let http_server_guard = test_util::http_server();
-    // Relies on external http server. See tools/http_server.py
+    // Relies on external http server. See target/debug/test_server
     let url =
       Url::parse("https://localhost:5545/cli/tests/fixture.json").unwrap();
 
@@ -425,7 +425,7 @@ mod tests {
   #[tokio::test]
   async fn test_fetch_with_cafile_gzip() {
     let http_server_guard = test_util::http_server();
-    // Relies on external http server. See tools/http_server.py
+    // Relies on external http server. See target/debug/test_server
     let url = Url::parse(
       "https://localhost:5545/cli/tests/053_import_compression/gziped",
     )
@@ -487,7 +487,7 @@ mod tests {
   #[tokio::test]
   async fn test_fetch_with_cafile_brotli() {
     let http_server_guard = test_util::http_server();
-    // Relies on external http server. See tools/http_server.py
+    // Relies on external http server. See target/debug/test_server
     let url = Url::parse(
       "https://localhost:5545/cli/tests/053_import_compression/brotli",
     )

cli/tests/053_import_compression/brotli.header

@@ -1,3 +0,0 @@
-Content-Encoding: br
-Content-Type: application/javascript
-Content-Length: 26

cli/tests/053_import_compression/gziped.header

@@ -1,3 +0,0 @@
-Content-Encoding: gzip
-Content-Type: application/javascript
-Content-Length: 39

cli/tests/integration_tests.rs

@@ -2600,7 +2600,8 @@ fn test_permissions_net_listen_allow_localhost_4555_fail() {
 
 #[test]
 fn test_permissions_net_listen_allow_localhost() {
-  // Port 4600 is chosen to not colide with those used by tools/http_server.py
+  // Port 4600 is chosen to not colide with those used by
+  // target/debug/test_server
   let (_, err) = util::run_and_collect_output(
     true,
 			"run --allow-net=localhost complex_permissions_test.ts netListen localhost:4600",

cli/tests/unit/README.md

@@ -75,6 +75,6 @@ RUST_BACKTRACE=1 cargo run -- run --unstable --allow-read --allow-write cli/test
 
 ### Http server
 
-`tools/http_server.py` is required to run when one's running unit tests. During
+`target/debug/test_server` is required to run when one's running unit tests.
-CI it's spawned automatically, but if you want to run tests manually make sure
+During CI it's spawned automatically, but if you want to run tests manually make
-that server is spawned otherwise there'll be cascade of test failures.
+sure that server is spawned otherwise there'll be cascade of test failures.

cli/tests/unit/body_test.ts

@@ -36,7 +36,7 @@ unitTest(
   { perms: { net: true } },
   async function bodyMultipartFormData(): Promise<void> {
     const response = await fetch(
-      "http://localhost:4545/cli/tests/subdir/multipart_form_data.txt"
+      "http://localhost:4545/multipart_form_data.txt"
     );
     const text = await response.text();
 

cli/tests/unit/fetch_test.ts

@@ -67,7 +67,6 @@ unitTest({ perms: { net: true } }, async function fetchHeaders(): Promise<
   const response = await fetch("http://localhost:4545/cli/tests/fixture.json");
   const headers = response.headers;
   assertEquals(headers.get("Content-Type"), "application/json");
-  assert(headers.get("Server")!.startsWith("SimpleHTTP"));
   const _json = await response.json();
 });
 
@@ -162,13 +161,10 @@ unitTest(
   { perms: { net: true } },
   async function fetchBodyReaderBigBody(): Promise<void> {
     const data = "a".repeat(10 << 10); // 10mb
-    const response = await fetch(
+    const response = await fetch("http://localhost:4545/echo_server", {
-      "http://localhost:4545/cli/tests/echo_server",
+      method: "POST",
-      {
+      body: data,
-        method: "POST",
+    });
-        body: data,
-      }
-    );
     assert(response.body !== null);
     const reader = await response.body.getReader();
     let total = 0;
@@ -210,7 +206,7 @@ unitTest(
   { perms: { net: true } },
   async function fetchMultipartFormDataSuccess(): Promise<void> {
     const response = await fetch(
-      "http://localhost:4545/cli/tests/subdir/multipart_form_data.txt"
+      "http://localhost:4545/multipart_form_data.txt"
     );
     const formData = await response.formData();
     assert(formData.has("field_1"));
@@ -315,12 +311,12 @@ unitTest(
     perms: { net: true },
   },
   async function fetchWithRedirection(): Promise<void> {
-    const response = await fetch("http://localhost:4546/"); // will redirect to http://localhost:4545/
+    const response = await fetch("http://localhost:4546/README.md");
     assertEquals(response.status, 200);
     assertEquals(response.statusText, "OK");
-    assertEquals(response.url, "http://localhost:4545/");
+    assertEquals(response.url, "http://localhost:4545/README.md");
     const body = await response.text();
-    assert(body.includes("<title>Directory listing for /</title>"));
+    assert(body.includes("Deno"));
   }
 );
 
@@ -329,11 +325,13 @@ unitTest(
     perms: { net: true },
   },
   async function fetchWithRelativeRedirection(): Promise<void> {
-    const response = await fetch("http://localhost:4545/cli/tests"); // will redirect to /cli/tests/
+    const response = await fetch(
+      "http://localhost:4545/cli/tests/001_hello.js"
+    );
     assertEquals(response.status, 200);
     assertEquals(response.statusText, "OK");
     const body = await response.text();
-    assert(body.includes("<title>Directory listing for /cli/tests/</title>"));
+    assert(body.includes("Hello"));
   }
 );
 
@@ -769,13 +767,10 @@ unitTest(
   { perms: { net: true } },
   async function fetchBodyReaderWithCancelAndNewReader(): Promise<void> {
     const data = "a".repeat(1 << 10);
-    const response = await fetch(
+    const response = await fetch("http://localhost:4545/echo_server", {
-      "http://localhost:4545/cli/tests/echo_server",
+      method: "POST",
-      {
+      body: data,
-        method: "POST",
+    });
-        body: data,
-      }
-    );
     assert(response.body !== null);
     const firstReader = await response.body.getReader();
 
@@ -801,13 +796,10 @@ unitTest(
   async function fetchBodyReaderWithReadCancelAndNewReader(): Promise<void> {
     const data = "a".repeat(1 << 10);
 
-    const response = await fetch(
+    const response = await fetch("http://localhost:4545/echo_server", {
-      "http://localhost:4545/cli/tests/echo_server",
+      method: "POST",
-      {
+      body: data,
-        method: "POST",
+    });
-        body: data,
-      }
-    );
     assert(response.body !== null);
     const firstReader = await response.body.getReader();
 
@@ -848,7 +840,7 @@ unitTest(
 
     for (const status of nullBodyStatus) {
       const headers = new Headers([["x-status", String(status)]]);
-      const res = await fetch("http://localhost:4545/cli/tests/echo_server", {
+      const res = await fetch("http://localhost:4545/echo_server", {
         body: "deno",
         method: "POST",
         headers,

cli/tests/x_deno_warning.js.header

@@ -1,2 +0,0 @@
-Content-Type: application/javascript
-X-Deno-Warning: foobar

test_util/Cargo.toml

@@ -5,10 +5,16 @@ authors = ["the Deno authors"]
 edition = "2018"
 publish = false
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[[bin]]
+name = "test_server"
+path = "src/test_server.rs"
 
 [dependencies]
+tokio = { version = "0.2.21", features = ["rt-core", "tcp", "udp", "uds", "process", "fs", "blocking", "sync", "io-std", "macros", "time"] }
+futures = { version = "0.3.5", features = ["compat", "io-compat"] }
+bytes = "0.5.5"
 lazy_static = "1.4.0"
 os_pipe = "0.9.2"
 regex = "1.3.9"
 tempfile = "3.1.0"
+warp = { version = "0.2.3", features = ["tls"] }

test_util/src/lib.rs

@@ -3,10 +3,12 @@
 #[macro_use]
 extern crate lazy_static;
 
+use futures::future::{self, FutureExt};
 use os_pipe::pipe;
 use regex::Regex;
 use std::io::Read;
 use std::io::Write;
+use std::mem::replace;
 use std::path::PathBuf;
 use std::process::Child;
 use std::process::Command;
@@ -15,6 +17,20 @@ use std::process::Stdio;
 use std::sync::Mutex;
 use std::sync::MutexGuard;
 use tempfile::TempDir;
+use warp::http::Uri;
+use warp::http::{HeaderValue, Response, StatusCode};
+use warp::hyper::Body;
+use warp::reply::with_header;
+use warp::reply::Reply;
+use warp::Filter;
+
+const PORT: u16 = 4545;
+const REDIRECT_PORT: u16 = 4546;
+const ANOTHER_REDIRECT_PORT: u16 = 4547;
+const DOUBLE_REDIRECTS_PORT: u16 = 4548;
+const INF_REDIRECTS_PORT: u16 = 4549;
+const REDIRECT_ABSOLUTE_PORT: u16 = 4550;
+const HTTPS_PORT: u16 = 5545;
 
 pub const PERMISSION_VARIANTS: [&str; 5] =
   ["read", "write", "env", "net", "run"];
@@ -54,27 +70,358 @@ pub fn deno_exe_path() -> PathBuf {
   p
 }
 
+pub fn test_server_path() -> PathBuf {
+  let mut p = target_dir().join("test_server");
+  if cfg!(windows) {
+    p.set_extension("exe");
+  }
+  p
+}
+
+#[tokio::main]
+pub async fn run_all_servers() {
+  let routes = warp::path::full().map(|path: warp::path::FullPath| {
+    let p = path.as_str();
+    assert_eq!(&p[0..1], "/");
+    let url = format!("http://localhost:{}{}", PORT, p);
+    let u = url.parse::<Uri>().unwrap();
+    warp::redirect(u)
+  });
+  let redirect_server_fut =
+    warp::serve(routes).bind(([127, 0, 0, 1], REDIRECT_PORT));
+
+  let routes = warp::path::full().map(|path: warp::path::FullPath| {
+    let p = path.as_str();
+    assert_eq!(&p[0..1], "/");
+    let url = format!("http://localhost:{}/cli/tests/subdir{}", PORT, p);
+    let u = url.parse::<Uri>().unwrap();
+    warp::redirect(u)
+  });
+  let another_redirect_server_fut =
+    warp::serve(routes).bind(([127, 0, 0, 1], ANOTHER_REDIRECT_PORT));
+
+  let routes = warp::path::full().map(|path: warp::path::FullPath| {
+    let p = path.as_str();
+    assert_eq!(&p[0..1], "/");
+    let url = format!("http://localhost:{}{}", REDIRECT_PORT, p);
+    let u = url.parse::<Uri>().unwrap();
+    warp::redirect(u)
+  });
+  let double_redirect_server_fut =
+    warp::serve(routes).bind(([127, 0, 0, 1], DOUBLE_REDIRECTS_PORT));
+
+  let routes = warp::path::full().map(|path: warp::path::FullPath| {
+    let p = path.as_str();
+    assert_eq!(&p[0..1], "/");
+    let url = format!("http://localhost:{}{}", INF_REDIRECTS_PORT, p);
+    let u = url.parse::<Uri>().unwrap();
+    warp::redirect(u)
+  });
+  let inf_redirect_server_fut =
+    warp::serve(routes).bind(([127, 0, 0, 1], INF_REDIRECTS_PORT));
+
+  // redirect server that redirect to absolute paths under same host
+  // redirects /REDIRECT/file_name to /file_name
+  let routes = warp::path("REDIRECT")
+    .and(warp::path::peek())
+    .map(|path: warp::path::Peek| {
+      let p = path.as_str();
+      let url = format!("/{}", p);
+      let u = url.parse::<Uri>().unwrap();
+      warp::redirect(u)
+    })
+    .or(
+      warp::any()
+        .and(warp::path::peek())
+        .and(warp::fs::dir(root_path()))
+        .map(custom_headers),
+    );
+  let absolute_redirect_server_fut =
+    warp::serve(routes).bind(([127, 0, 0, 1], REDIRECT_ABSOLUTE_PORT));
+
+  let echo_server = warp::path("echo_server")
+    .and(warp::post())
+    .and(warp::body::bytes())
+    .and(warp::header::optional::<String>("x-status"))
+    .and(warp::header::optional::<String>("content-type"))
+    .and(warp::header::optional::<String>("user-agent"))
+    .map(
+      |bytes: bytes::Bytes,
+       status: Option<String>,
+       content_type: Option<String>,
+       user_agent: Option<String>|
+       -> Box<dyn Reply> {
+        let mut res = Response::new(Body::from(bytes));
+        if let Some(v) = status {
+          *res.status_mut() = StatusCode::from_bytes(v.as_bytes()).unwrap();
+        }
+        let h = res.headers_mut();
+        if let Some(v) = content_type {
+          h.insert("content-type", HeaderValue::from_str(&v).unwrap());
+        }
+        if let Some(v) = user_agent {
+          h.insert("user-agent", HeaderValue::from_str(&v).unwrap());
+        }
+        Box::new(res)
+      },
+    );
+  let echo_multipart_file = warp::path("echo_multipart_file")
+    .and(warp::post())
+    .and(warp::body::bytes())
+    .map(|bytes: bytes::Bytes| -> Box<dyn Reply> {
+      let start = b"--boundary\t \r\n\
+                    Content-Disposition: form-data; name=\"field_1\"\r\n\
+                    \r\n\
+                    value_1 \r\n\
+                    \r\n--boundary\r\n\
+                    Content-Disposition: form-data; name=\"file\"; \
+                    filename=\"file.bin\"\r\n\
+                    Content-Type: application/octet-stream\r\n\
+                    \r\n";
+      let end = b"\r\n--boundary--\r\n";
+      let b = [start as &[u8], &bytes, end].concat();
+
+      let mut res = Response::new(Body::from(b));
+      let h = res.headers_mut();
+      h.insert(
+        "content-type",
+        HeaderValue::from_static("multipart/form-data;boundary=boundary"),
+      );
+      Box::new(res)
+    });
+  let multipart_form_data =
+    warp::path("multipart_form_data.txt").map(|| -> Box<dyn Reply> {
+      let b = "Preamble\r\n\
+               --boundary\t \r\n\
+               Content-Disposition: form-data; name=\"field_1\"\r\n\
+               \r\n\
+               value_1 \r\n\
+               \r\n--boundary\r\n\
+               Content-Disposition: form-data; name=\"field_2\";\
+               filename=\"file.js\"\r\n\
+               Content-Type: text/javascript\r\n\
+               \r\n\
+               console.log(\"Hi\")\
+               \r\n--boundary--\r\n\
+               Epilogue";
+      let mut res = Response::new(Body::from(b));
+      res.headers_mut().insert(
+        "content-type",
+        HeaderValue::from_static("multipart/form-data;boundary=boundary"),
+      );
+      Box::new(res)
+    });
+
+  let etag_script = warp::path!("etag_script.ts")
+    .and(warp::header::optional::<String>("if-none-match"))
+    .map(|if_none_match| -> Box<dyn Reply> {
+      if if_none_match == Some("33a64df551425fcc55e".to_string()) {
+        let r =
+          warp::reply::with_status(warp::reply(), StatusCode::NOT_MODIFIED);
+        let r = with_header(r, "Content-type", "application/typescript");
+        let r = with_header(r, "ETag", "33a64df551425fcc55e");
+        Box::new(r)
+      } else {
+        let mut res = Response::new(Body::from("console.log('etag')"));
+        let h = res.headers_mut();
+        h.insert(
+          "Content-type",
+          HeaderValue::from_static("application/typescript"),
+        );
+        h.insert("ETag", HeaderValue::from_static("33a64df551425fcc55e"));
+        Box::new(res)
+      }
+    });
+  let xtypescripttypes = warp::path!("xTypeScriptTypes.js")
+    .map(|| {
+      let mut res = Response::new(Body::from("export const foo = 'foo';"));
+      let h = res.headers_mut();
+      h.insert(
+        "Content-type",
+        HeaderValue::from_static("application/javascript"),
+      );
+      h.insert(
+        "X-TypeScript-Types",
+        HeaderValue::from_static("./xTypeScriptTypes.d.ts"),
+      );
+      res
+    })
+    .or(warp::path!("xTypeScriptTypes.d.ts").map(|| {
+      let mut res = Response::new(Body::from("export const foo: 'foo';"));
+      res.headers_mut().insert(
+        "Content-type",
+        HeaderValue::from_static("application/typescript"),
+      );
+      res
+    }))
+    .or(warp::path!("type_directives_redirect.js").map(|| {
+      let mut res = Response::new(Body::from("export const foo = 'foo';"));
+      let h = res.headers_mut();
+      h.insert(
+        "Content-type",
+        HeaderValue::from_static("application/javascript"),
+      );
+      h.insert(
+        "X-TypeScript-Types",
+        HeaderValue::from_static(
+          "http://localhost:4547/xTypeScriptTypesRedirect.d.ts",
+        ),
+      );
+      res
+    }))
+    .or(warp::path!("cli"/"tests"/"subdir"/"xTypeScriptTypesRedirect.d.ts").map(|| {
+      let mut res = Response::new(Body::from(
+        "import './xTypeScriptTypesRedirected.d.ts';",
+      ));
+      let h = res.headers_mut();
+      h.insert(
+        "Content-type",
+        HeaderValue::from_static("application/typescript"),
+      );
+      res
+    }))
+    .or(warp::path!("cli"/"tests"/"subdir"/"xTypeScriptTypesRedirected.d.ts").map(|| {
+      let mut res = Response::new(Body::from("export const foo: 'foo';"));
+      let h = res.headers_mut();
+      h.insert(
+        "Content-type",
+        HeaderValue::from_static("application/typescript"),
+      );
+      res
+    }))
+    .or(warp::path!("referenceTypes.js").map(|| {
+      let mut res = Response::new(Body::from("/// <reference types=\"./xTypeScriptTypes.d.ts\" />\r\nexport const foo = \"foo\";\r\n"));
+      let h = res.headers_mut();
+      h.insert(
+        "Content-type",
+        HeaderValue::from_static("application/javascript"),
+      );
+      res
+    }));
+
+  let content_type_handler = warp::any()
+    .and(warp::path::peek())
+    .and(warp::fs::dir(root_path()))
+    .map(custom_headers)
+    .or(etag_script)
+    .or(xtypescripttypes)
+    .or(echo_server)
+    .or(echo_multipart_file)
+    .or(multipart_form_data);
+
+  let http_fut =
+    warp::serve(content_type_handler.clone()).bind(([127, 0, 0, 1], PORT));
+
+  let https_fut = warp::serve(content_type_handler.clone())
+    .tls()
+    .cert_path("std/http/testdata/tls/localhost.crt")
+    .key_path("std/http/testdata/tls/localhost.key")
+    .bind(([127, 0, 0, 1], HTTPS_PORT));
+
+  let mut server_fut = async {
+    futures::join!(
+      http_fut,
+      https_fut,
+      redirect_server_fut,
+      another_redirect_server_fut,
+      inf_redirect_server_fut,
+      double_redirect_server_fut,
+      absolute_redirect_server_fut,
+    )
+  }
+  .boxed();
+
+  let mut did_print_ready = false;
+  future::poll_fn(move |cx| {
+    let poll_result = server_fut.poll_unpin(cx);
+    if !replace(&mut did_print_ready, true) {
+      println!("ready");
+    }
+    poll_result
+  })
+  .await;
+}
+
+fn custom_headers(path: warp::path::Peek, f: warp::fs::File) -> Box<dyn Reply> {
+  let p = path.as_str();
+
+  if p.ends_with("cli/tests/x_deno_warning.js") {
+    let f = with_header(f, "Content-Type", "application/javascript");
+    let f = with_header(f, "X-Deno-Warning", "foobar");
+    return Box::new(f);
+  }
+  if p.ends_with("cli/tests/053_import_compression/brotli") {
+    let f = with_header(f, "Content-Encoding", "br");
+    let f = with_header(f, "Content-Type", "application/javascript");
+    let f = with_header(f, "Content-Length", "26");
+    return Box::new(f);
+  }
+  if p.ends_with("cli/tests/053_import_compression/gziped") {
+    let f = with_header(f, "Content-Encoding", "gzip");
+    let f = with_header(f, "Content-Type", "application/javascript");
+    let f = with_header(f, "Content-Length", "39");
+    return Box::new(f);
+  }
+
+  let content_type = if p.contains(".t1.") {
+    Some("text/typescript")
+  } else if p.contains(".t2.") {
+    Some("video/vnd.dlna.mpeg-tts")
+  } else if p.contains(".t3.") {
+    Some("video/mp2t")
+  } else if p.contains(".t4.") {
+    Some("application/x-typescript")
+  } else if p.contains(".j1.") {
+    Some("text/javascript")
+  } else if p.contains(".j2.") {
+    Some("application/ecmascript")
+  } else if p.contains(".j3.") {
+    Some("text/ecmascript")
+  } else if p.contains(".j4.") {
+    Some("application/x-javascript")
+  } else if p.contains("form_urlencoded") {
+    Some("application/x-www-form-urlencoded")
+  } else if p.contains("unknown_ext") || p.contains("no_ext") {
+    Some("text/typescript")
+  } else if p.contains("mismatch_ext") {
+    Some("text/javascript")
+  } else if p.ends_with(".ts") || p.ends_with(".tsx") {
+    Some("application/typescript")
+  } else if p.ends_with(".js") || p.ends_with(".jsx") {
+    Some("application/javascript")
+  } else if p.ends_with(".json") {
+    Some("application/json")
+  } else {
+    None
+  };
+
+  if let Some(t) = content_type {
+    Box::new(with_header(f, "Content-Type", t))
+  } else {
+    Box::new(f)
+  }
+}
+
 pub struct HttpServerGuard<'a> {
   #[allow(dead_code)]
   g: MutexGuard<'a, ()>,
-  child: Child,
+  test_server: Child,
 }
 
 impl<'a> Drop for HttpServerGuard<'a> {
   fn drop(&mut self) {
-    match self.child.try_wait() {
+    match self.test_server.try_wait() {
       Ok(None) => {
-        self.child.kill().expect("failed to kill http_server.py");
+        self.test_server.kill().expect("failed to kill test_server");
+        let _ = self.test_server.wait();
       }
-      Ok(Some(status)) => {
+      Ok(Some(status)) => panic!("test_server exited unexpectedly {}", status),
-        panic!("http_server.py exited unexpectedly {}", status)
+      Err(e) => panic!("test_server error: {}", e),
-      }
-      Err(e) => panic!("http_server.py err {}", e),
     }
   }
 }
 
-/// Starts tools/http_server.py when the returned guard is dropped, the server
+/// Starts target/debug/test_server when the returned guard is dropped, the server
 /// will be killed.
 pub fn http_server<'a>() -> HttpServerGuard<'a> {
   // TODO(bartlomieju) Allow tests to use the http server in parallel.
@@ -86,18 +433,16 @@ pub fn http_server<'a>() -> HttpServerGuard<'a> {
     r.unwrap()
   };
 
-  println!("tools/http_server.py starting...");
+  println!("test_server starting...");
-  let mut child = Command::new("python")
+  let mut test_server = Command::new(test_server_path())
     .current_dir(root_path())
-    .args(&["-u", "tools/http_server.py"])
     .stdout(Stdio::piped())
     .spawn()
-    .expect("failed to execute child");
+    .expect("failed to execute test_server");
 
-  let stdout = child.stdout.as_mut().unwrap();
+  let stdout = test_server.stdout.as_mut().unwrap();
   use std::io::{BufRead, BufReader};
   let lines = BufReader::new(stdout).lines();
-  // Wait for "ready" on stdout. See tools/http_server.py
   for maybe_line in lines {
     if let Ok(line) = maybe_line {
       if line.starts_with("ready") {
@@ -108,7 +453,7 @@ pub fn http_server<'a>() -> HttpServerGuard<'a> {
     }
   }
 
-  HttpServerGuard { child, g }
+  HttpServerGuard { test_server, g }
 }
 
 /// Helper function to strip ansi codes.

test_util/src/test_server.rs

@@ -0,0 +1,3 @@
+fn main() {
+  test_util::run_all_servers();
+}

tools/benchmark.py

@@ -3,7 +3,7 @@
 # Performs benchmark and append data to //website/data.json.
 # If //website/data.json doesn't exist, this script tries to import it from
 # gh-pages branch.
-# To view the results locally run ./tools/http_server.py and visit
+# To view the results locally run target/debug/test_server and visit
 # http://localhost:4545/website
 
 import os
@@ -12,11 +12,11 @@ import json
 import time
 import tempfile
 import subprocess
-from util import build_path, executable_suffix, root_path, run, run_output
+from util import (build_path, executable_suffix, root_path, run, run_output,
+                  build_mode)
 import third_party
 from http_benchmark import http_benchmark
 import throughput_benchmark
-import http_server
 
 # The list of the tuples of the benchmark name, arguments and return code
 exec_time_benchmarks = [
@@ -239,7 +239,6 @@ def main():
     build_dir = build_path()
     sha1 = run_output(["git", "rev-parse", "HEAD"],
                       exit_on_fail=True).out.strip()
-    http_server.spawn()
 
     deno_exe = os.path.join(build_dir, "deno")
 
@@ -253,7 +252,11 @@ def main():
     # TODO(ry) The "benchmark" benchmark should actually be called "exec_time".
     # When this is changed, the historical data in gh-pages branch needs to be
     # changed too.
+    server_cmd = os.path.join("target", build_mode(), "test_server")
+    p = subprocess.Popen([server_cmd])
     new_data["benchmark"] = run_exec_time(deno_exe, build_dir)
+    p.kill()
+    p.wait()
 
     new_data["binary_size"] = get_binary_sizes(build_dir)
     new_data["bundle_size"] = bundle_benchmark(deno_exe)

tools/benchmark_test.py

@@ -65,6 +65,4 @@ class TestBenchmark(DenoTestCase):
 
 
 if __name__ == '__main__':
-    # FIME this doesn't appear to be the case.
-    # This test assumes tools/http_server.py is running in the background.
     run_tests()

tools/http_server.py

@@ -1,420 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
-# Many tests expect there to be an http server on port 4545 servering the deno
-# root directory.
-from collections import namedtuple
-from contextlib import contextmanager
-import os
-import SimpleHTTPServer
-import SocketServer
-import socket
-import sys
-from time import sleep
-from threading import Thread
-from util import root_path
-import ssl
-import getopt
-import argparse
-
-PORT = 4545
-REDIRECT_PORT = 4546
-ANOTHER_REDIRECT_PORT = 4547
-DOUBLE_REDIRECTS_PORT = 4548
-INF_REDIRECTS_PORT = 4549
-REDIRECT_ABSOLUTE_PORT = 4550
-HTTPS_PORT = 5545
-
-
-def create_http_arg_parser():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--verbose', '-v', action='store_true')
-    return parser
-
-
-HttpArgParser = create_http_arg_parser()
-
-args, unknown = HttpArgParser.parse_known_args(sys.argv[1:])
-CERT_FILE = os.path.join(root_path, "std/http/testdata/tls/localhost.crt")
-KEY_FILE = os.path.join(root_path, "std/http/testdata/tls/localhost.key")
-QUIET = not args.verbose
-
-
-class SSLTCPServer(SocketServer.TCPServer):
-    def __init__(self,
-                 server_address,
-                 request_handler,
-                 certfile,
-                 keyfile,
-                 ssl_version=ssl.PROTOCOL_TLSv1_2,
-                 bind_and_activate=True):
-        SocketServer.TCPServer.__init__(self, server_address, request_handler,
-                                        bind_and_activate)
-        self.certfile = certfile
-        self.keyfile = keyfile
-        self.ssl_version = ssl_version
-
-    def get_request(self):
-        newsocket, fromaddr = self.socket.accept()
-        connstream = ssl.wrap_socket(
-            newsocket,
-            server_side=True,
-            certfile=self.certfile,
-            keyfile=self.keyfile,
-            ssl_version=self.ssl_version)
-        return connstream, fromaddr
-
-
-class SSLThreadingTCPServer(SocketServer.ThreadingMixIn, SSLTCPServer):
-    pass
-
-
-class QuietSimpleHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
-    def log_request(self, code='-', size='-'):
-        if not QUIET:
-            SimpleHTTPServer.SimpleHTTPRequestHandler.log_request(
-                self, code, size)
-
-
-class ContentTypeHandler(QuietSimpleHTTPRequestHandler):
-    def do_GET(self):
-
-        # Check if there is a custom header configuration ending
-        # with ".header" before sending the file
-        maybe_header_file_path = "./" + self.path + ".header"
-        if os.path.exists(maybe_header_file_path):
-            self.protocol_version = 'HTTP/1.1'
-            self.send_response(200, 'OK')
-
-            f = open(maybe_header_file_path)
-            for line in f:
-                kv = line.split(": ")
-                self.send_header(kv[0].strip(), kv[1].strip())
-            f.close()
-            self.end_headers()
-
-            body = open("./" + self.path)
-            self.wfile.write(body.read())
-            body.close()
-            return
-
-        if "etag_script.ts" in self.path:
-            self.protocol_version = 'HTTP/1.1'
-            if_not_match = self.headers.getheader('if-none-match')
-            if if_not_match == "33a64df551425fcc55e":
-                self.send_response(304, 'Not Modified')
-                self.send_header('Content-type', 'application/typescript')
-                self.send_header('ETag', '33a64df551425fcc55e')
-                self.end_headers()
-            else:
-                self.send_response(200, 'OK')
-                self.send_header('Content-type', 'application/typescript')
-                self.send_header('ETag', '33a64df551425fcc55e')
-                self.end_headers()
-                self.wfile.write(bytes("console.log('etag')"))
-            return
-
-        if "xTypeScriptTypes.js" in self.path:
-            self.protocol_version = "HTTP/1.1"
-            self.send_response(200, 'OK')
-            self.send_header('Content-type', 'application/javascript')
-            self.send_header('X-TypeScript-Types', './xTypeScriptTypes.d.ts')
-            self.end_headers()
-            self.wfile.write(bytes("export const foo = 'foo';"))
-            return
-
-        if "type_directives_redirect.js" in self.path:
-            self.protocol_version = "HTTP/1.1"
-            self.send_response(200, 'OK')
-            self.send_header('Content-type', 'application/javascript')
-            self.send_header(
-                'X-TypeScript-Types',
-                'http://localhost:4547/xTypeScriptTypesRedirect.d.ts')
-            self.end_headers()
-            self.wfile.write(bytes("export const foo = 'foo';"))
-            return
-
-        if "xTypeScriptTypesRedirect.d.ts" in self.path:
-            self.protocol_version = "HTTP/1.1"
-            self.send_response(200, 'OK')
-            self.send_header('Content-type', 'application/typescript')
-            self.end_headers()
-            self.wfile.write(
-                bytes("import './xTypeScriptTypesRedirected.d.ts';"))
-            return
-
-        if "xTypeScriptTypesRedirected.d.ts" in self.path:
-            self.protocol_version = "HTTP/1.1"
-            self.send_response(200, 'OK')
-            self.send_header('Content-type', 'application/typescript')
-            self.end_headers()
-            self.wfile.write(bytes("export const foo: 'foo';"))
-            return
-
-        if "xTypeScriptTypes.d.ts" in self.path:
-            self.protocol_version = "HTTP/1.1"
-            self.send_response(200, 'OK')
-            self.send_header('Content-type', 'application/typescript')
-            self.end_headers()
-            self.wfile.write(bytes("export const foo: 'foo';"))
-            return
-
-        if "referenceTypes.js" in self.path:
-            self.protocol_version = "HTTP/1.1"
-            self.send_response(200, 'OK')
-            self.send_header('Content-type', 'application/javascript')
-            self.end_headers()
-            self.wfile.write(
-                bytes('/// <reference types="./xTypeScriptTypes.d.ts" />\r\n'
-                      'export const foo = "foo";\r\n'))
-            return
-
-        if "multipart_form_data.txt" in self.path:
-            self.protocol_version = 'HTTP/1.1'
-            self.send_response(200, 'OK')
-            self.send_header('Content-type',
-                             'multipart/form-data;boundary=boundary')
-            self.end_headers()
-            self.wfile.write(
-                bytes('Preamble\r\n'
-                      '--boundary\t \r\n'
-                      'Content-Disposition: form-data; name="field_1"\r\n'
-                      '\r\n'
-                      'value_1 \r\n'
-                      '\r\n--boundary\r\n'
-                      'Content-Disposition: form-data; name="field_2"; '
-                      'filename="file.js"\r\n'
-                      'Content-Type: text/javascript\r\n'
-                      '\r\n'
-                      'console.log("Hi")'
-                      '\r\n--boundary--\r\n'
-                      'Epilogue'))
-            return
-        return SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self)
-
-    def do_POST(self):
-        # Simple echo server for request reflection
-        if "echo_server" in self.path:
-            status = int(self.headers.getheader('x-status', "200"))
-            self.protocol_version = 'HTTP/1.1'
-            self.send_response(status, 'OK')
-            if self.headers.has_key('content-type'):
-                self.send_header('content-type',
-                                 self.headers.getheader('content-type'))
-            if self.headers.has_key('user-agent'):
-                self.send_header('user-agent',
-                                 self.headers.getheader('user-agent'))
-            self.end_headers()
-            data_string = self.rfile.read(int(self.headers['Content-Length']))
-            self.wfile.write(bytes(data_string))
-            return
-        if "echo_multipart_file" in self.path:
-            self.protocol_version = 'HTTP/1.1'
-            self.send_response(200, 'OK')
-            self.send_header('Content-type',
-                             'multipart/form-data;boundary=boundary')
-            self.end_headers()
-            file_content = self.rfile.read(int(self.headers['Content-Length']))
-            self.wfile.write(
-                bytes('--boundary\t \r\n'
-                      'Content-Disposition: form-data; name="field_1"\r\n'
-                      '\r\n'
-                      'value_1 \r\n'
-                      '\r\n--boundary\r\n'
-                      'Content-Disposition: form-data; name="file"; '
-                      'filename="file.bin"\r\n'
-                      'Content-Type: application/octet-stream\r\n'
-                      '\r\n') + bytes(file_content) +
-                bytes('\r\n--boundary--\r\n'))
-            return
-        self.protocol_version = 'HTTP/1.1'
-        self.send_response(501)
-        self.send_header('content-type', 'text/plain')
-        self.end_headers()
-        self.wfile.write(bytes('Server does not support this operation'))
-
-    def guess_type(self, path):
-        if ".t1." in path:
-            return "text/typescript"
-        if ".t2." in path:
-            return "video/vnd.dlna.mpeg-tts"
-        if ".t3." in path:
-            return "video/mp2t"
-        if ".t4." in path:
-            return "application/x-typescript"
-        if ".j1." in path:
-            return "text/javascript"
-        if ".j2." in path:
-            return "application/ecmascript"
-        if ".j3." in path:
-            return "text/ecmascript"
-        if ".j4." in path:
-            return "application/x-javascript"
-        if "form_urlencoded" in path:
-            return "application/x-www-form-urlencoded"
-        if "no_ext" in path:
-            return "text/typescript"
-        if "unknown_ext" in path:
-            return "text/typescript"
-        if "mismatch_ext" in path:
-            return "text/javascript"
-        return SimpleHTTPServer.SimpleHTTPRequestHandler.guess_type(self, path)
-
-
-RunningServer = namedtuple("RunningServer", ["server", "thread"])
-
-
-def get_socket(port, handler, use_https):
-    SocketServer.TCPServer.allow_reuse_address = True
-    if os.name != "nt":
-        # We use AF_INET6 to avoid flaky test issue, particularly with
-        # the test 019_media_types. It's not well understood why this fixes the
-        # flaky tests, but it does appear to...
-        # See https://github.com/denoland/deno/issues/3332
-        SocketServer.TCPServer.address_family = socket.AF_INET6
-
-    if use_https:
-        return SSLThreadingTCPServer(("", port), handler, CERT_FILE, KEY_FILE)
-    return SocketServer.TCPServer(("", port), handler)
-
-
-def server():
-    os.chdir(root_path)  # Hopefully the main thread doesn't also chdir.
-    Handler = ContentTypeHandler
-    Handler.extensions_map.update({
-        ".ts": "application/typescript",
-        ".js": "application/javascript",
-        ".tsx": "application/typescript",
-        ".jsx": "application/javascript",
-        ".json": "application/json",
-    })
-    s = get_socket(PORT, Handler, False)
-    if not QUIET:
-        print "Deno test server http://localhost:%d/" % PORT
-    return RunningServer(s, start(s))
-
-
-def base_redirect_server(host_port, target_port, extra_path_segment=""):
-    os.chdir(root_path)
-    target_host = "http://localhost:%d" % target_port
-
-    class RedirectHandler(QuietSimpleHTTPRequestHandler):
-        def do_GET(self):
-            self.send_response(301)
-            self.send_header('Location',
-                             target_host + extra_path_segment + self.path)
-            self.end_headers()
-
-    s = get_socket(host_port, RedirectHandler, False)
-    if not QUIET:
-        print "redirect server http://localhost:%d/ -> http://localhost:%d/" % (
-            host_port, target_port)
-    return RunningServer(s, start(s))
-
-
-# redirect server
-def redirect_server():
-    return base_redirect_server(REDIRECT_PORT, PORT)
-
-
-# another redirect server pointing to the same port as the one above
-# BUT with an extra subdir path
-def another_redirect_server():
-    return base_redirect_server(
-        ANOTHER_REDIRECT_PORT, PORT, extra_path_segment="/cli/tests/subdir")
-
-
-# redirect server that points to another redirect server
-def double_redirects_server():
-    return base_redirect_server(DOUBLE_REDIRECTS_PORT, REDIRECT_PORT)
-
-
-# redirect server that points to itself
-def inf_redirects_server():
-    return base_redirect_server(INF_REDIRECTS_PORT, INF_REDIRECTS_PORT)
-
-
-# redirect server that redirect to absolute paths under same host
-# redirects /REDIRECT/file_name to /file_name
-def absolute_redirect_server():
-    os.chdir(root_path)
-
-    class AbsoluteRedirectHandler(ContentTypeHandler):
-        def do_GET(self):
-            print(self.path)
-            if (self.path.startswith("/REDIRECT/")):
-                self.send_response(302)
-                self.send_header('Location',
-                                 self.path.split('/REDIRECT', 1)[1])
-                self.end_headers()
-            else:
-                ContentTypeHandler.do_GET(self)
-
-    s = get_socket(REDIRECT_ABSOLUTE_PORT, AbsoluteRedirectHandler, False)
-    if not QUIET:
-        print("absolute redirect server http://localhost:%d/" %
-              REDIRECT_ABSOLUTE_PORT)
-    return RunningServer(s, start(s))
-
-
-def https_server():
-    os.chdir(root_path)  # Hopefully the main thread doesn't also chdir.
-    Handler = ContentTypeHandler
-    Handler.extensions_map.update({
-        ".ts": "application/typescript",
-        ".js": "application/javascript",
-        ".tsx": "application/typescript",
-        ".jsx": "application/javascript",
-        ".json": "application/json",
-    })
-    s = get_socket(HTTPS_PORT, Handler, True)
-    if not QUIET:
-        print "Deno https test server https://localhost:%d/" % HTTPS_PORT
-    return RunningServer(s, start(s))
-
-
-def start(s):
-    thread = Thread(target=s.serve_forever, kwargs={"poll_interval": 0.05})
-    thread.daemon = True
-    thread.start()
-    return thread
-
-
-@contextmanager
-def spawn():
-    servers = (server(), redirect_server(), another_redirect_server(),
-               double_redirects_server(), https_server(),
-               absolute_redirect_server(), inf_redirects_server())
-    # In order to wait for each of the servers to be ready, we try connecting to
-    # them with a tcp socket.
-    for running_server in servers:
-        client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        port = running_server.server.server_address[1]
-        client.connect(("127.0.0.1", port))
-        print "connected", port
-        client.close()
-        assert running_server.thread.is_alive()
-    # The following output "ready" is specificly looked for in cli/test_util.rs
-    # to prevent race conditions.
-    print "ready"
-    try:
-        yield servers
-    finally:
-        for s in servers:
-            # Make sure all servers still running,
-            # if not assume there was an error
-            assert s.thread.is_alive()
-            s.server.shutdown()
-
-
-def main():
-    with spawn() as servers:
-        try:
-            while all(s.thread.is_alive() for s in servers):
-                sleep(1)
-        except KeyboardInterrupt:
-            pass
-    sys.exit(1)
-
-
-if __name__ == '__main__':
-    main()

tools/throughput_benchmark.py

@@ -3,7 +3,7 @@
 # Performs benchmark and append data to //website/data.json.
 # If //website/data.json doesn't exist, this script tries to import it from
 # gh-pages branch.
-# To view the results locally run ./tools/http_server.py and visit
+# To view the results locally run target/debug/test_server and visit
 # http://localhost:4545/website
 
 import os