mirror of
https://github.com/denoland/deno.git
synced 2025-08-03 18:38:33 +00:00
Bartek Iwańczuk
Bert Belder
parent
840c4aa2b2
commit
723284fd20
7 changed files with 175 additions and 282 deletions
301
cli/http_util.rs
301
cli/http_util.rs
|
|
@ -2,76 +2,56 @@
|
|||
use crate::deno_error;
|
||||
use crate::deno_error::DenoError;
|
||||
use deno::ErrBox;
|
||||
#[cfg(test)]
|
||||
use futures::future::{loop_fn, Loop};
|
||||
use futures::{future, Future, Stream};
|
||||
use hyper;
|
||||
use hyper::client::{Client, HttpConnector};
|
||||
use hyper::header::CONTENT_TYPE;
|
||||
use hyper::Uri;
|
||||
use hyper_rustls;
|
||||
use futures::{future, Future};
|
||||
use reqwest;
|
||||
use reqwest::header::CONTENT_TYPE;
|
||||
use reqwest::header::LOCATION;
|
||||
use reqwest::r#async::Client;
|
||||
use reqwest::RedirectPolicy;
|
||||
use url::Url;
|
||||
|
||||
type Connector = hyper_rustls::HttpsConnector<HttpConnector>;
|
||||
|
||||
lazy_static! {
|
||||
static ref CONNECTOR: Connector = {
|
||||
let num_dns_threads = 4;
|
||||
Connector::new(num_dns_threads)
|
||||
};
|
||||
}
|
||||
|
||||
pub fn get_client() -> Client<Connector, hyper::Body> {
|
||||
// TODO use Hyper's connection pool.
|
||||
let c = CONNECTOR.clone();
|
||||
Client::builder().build(c)
|
||||
/// Create new instance of async reqwest::Client. This client supports
|
||||
/// proxies and doesn't follow redirects.
|
||||
pub fn get_client() -> Client {
|
||||
Client::builder()
|
||||
.redirect(RedirectPolicy::none())
|
||||
.use_sys_proxy()
|
||||
.build()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
/// Construct the next uri based on base uri and location header fragment
|
||||
/// See <https://tools.ietf.org/html/rfc3986#section-4.2>
|
||||
fn resolve_uri_from_location(base_uri: &Uri, location: &str) -> Uri {
|
||||
fn resolve_url_from_location(base_url: &Url, location: &str) -> Url {
|
||||
if location.starts_with("http://") || location.starts_with("https://") {
|
||||
// absolute uri
|
||||
location
|
||||
.parse::<Uri>()
|
||||
.expect("provided redirect url should be a valid url")
|
||||
Url::parse(location).expect("provided redirect url should be a valid url")
|
||||
} else if location.starts_with("//") {
|
||||
// "//" authority path-abempty
|
||||
format!("{}:{}", base_uri.scheme_part().unwrap().as_str(), location)
|
||||
.parse::<Uri>()
|
||||
Url::parse(&format!("{}:{}", base_url.scheme(), location))
|
||||
.expect("provided redirect url should be a valid url")
|
||||
} else if location.starts_with('/') {
|
||||
// path-absolute
|
||||
let mut new_uri_parts = base_uri.clone().into_parts();
|
||||
new_uri_parts.path_and_query = Some(location.parse().unwrap());
|
||||
Uri::from_parts(new_uri_parts).unwrap()
|
||||
base_url
|
||||
.join(location)
|
||||
.expect("provided redirect url should be a valid url")
|
||||
} else {
|
||||
// assuming path-noscheme | path-empty
|
||||
let mut new_uri_parts = base_uri.clone().into_parts();
|
||||
let base_uri_path_str = base_uri.path().to_owned();
|
||||
let segs: Vec<&str> = base_uri_path_str.rsplitn(2, '/').collect();
|
||||
new_uri_parts.path_and_query = Some(
|
||||
format!("{}/{}", segs.last().unwrap_or(&""), location)
|
||||
.parse()
|
||||
.unwrap(),
|
||||
);
|
||||
Uri::from_parts(new_uri_parts).unwrap()
|
||||
let base_url_path_str = base_url.path().to_owned();
|
||||
// Pop last part or url (after last slash)
|
||||
let segs: Vec<&str> = base_url_path_str.rsplitn(2, '/').collect();
|
||||
let new_path = format!("{}/{}", segs.last().unwrap_or(&""), location);
|
||||
base_url
|
||||
.join(&new_path)
|
||||
.expect("provided redirect url should be a valid url")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
use crate::tokio_util;
|
||||
#[cfg(test)]
|
||||
/// Synchronously fetchs the given HTTP URL. Returns (content, media_type).
|
||||
pub fn fetch_sync_string(
|
||||
module_name: &str,
|
||||
) -> Result<(String, String), ErrBox> {
|
||||
tokio_util::block_on(fetch_string(module_name))
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq)]
|
||||
pub enum FetchOnceResult {
|
||||
// (code, maybe_content_type)
|
||||
Code(String, Option<String>),
|
||||
Redirect(http::uri::Uri),
|
||||
Redirect(Url),
|
||||
}
|
||||
|
||||
/// Asynchronously fetchs the given HTTP URL one pass only.
|
||||
|
|
@ -80,51 +60,57 @@ pub enum FetchOnceResult {
|
|||
/// If redirect occurs, does not follow and
|
||||
/// yields Redirect(url).
|
||||
pub fn fetch_string_once(
|
||||
url: http::uri::Uri,
|
||||
url: &Url,
|
||||
) -> impl Future<Item = FetchOnceResult, Error = ErrBox> {
|
||||
type FetchAttempt = (Option<String>, Option<String>, Option<FetchOnceResult>);
|
||||
|
||||
let url = url.clone();
|
||||
let client = get_client();
|
||||
|
||||
client
|
||||
.get(url.clone())
|
||||
.send()
|
||||
.map_err(ErrBox::from)
|
||||
.and_then(
|
||||
move |response| -> Box<
|
||||
move |mut response| -> Box<
|
||||
dyn Future<Item = FetchAttempt, Error = ErrBox> + Send,
|
||||
> {
|
||||
if response.status().is_redirection() {
|
||||
let location_string = response
|
||||
.headers()
|
||||
.get("location")
|
||||
let location_string = response.headers()
|
||||
.get(LOCATION)
|
||||
.expect("url redirection should provide 'location' header")
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_string();
|
||||
debug!("Redirecting to {}...", &location_string);
|
||||
let new_url = resolve_uri_from_location(&url, &location_string);
|
||||
.unwrap();
|
||||
|
||||
debug!("Redirecting to {:?}...", &location_string);
|
||||
let new_url = resolve_url_from_location(&url, location_string);
|
||||
// Boxed trait object turns out to be the savior for 2+ types yielding same results.
|
||||
return Box::new(future::ok(None).join3(
|
||||
future::ok(None),
|
||||
future::ok(Some(FetchOnceResult::Redirect(new_url))),
|
||||
));
|
||||
} else if response.status().is_client_error()
|
||||
|| response.status().is_server_error()
|
||||
{
|
||||
}
|
||||
|
||||
if response.status().is_client_error() || response.status().is_server_error() {
|
||||
return Box::new(future::err(DenoError::new(
|
||||
deno_error::ErrorKind::Other,
|
||||
format!("Import '{}' failed: {}", &url, response.status()),
|
||||
).into()));
|
||||
}
|
||||
|
||||
let content_type = response
|
||||
.headers()
|
||||
.get(CONTENT_TYPE)
|
||||
.map(|content_type| content_type.to_str().unwrap().to_owned());
|
||||
|
||||
let body = response
|
||||
.into_body()
|
||||
.concat2()
|
||||
.map(|body| String::from_utf8(body.to_vec()).ok())
|
||||
.text()
|
||||
.map_err(ErrBox::from);
|
||||
Box::new(body.join3(future::ok(content_type), future::ok(None)))
|
||||
},
|
||||
|
||||
Box::new(
|
||||
Some(body).join3(future::ok(content_type), future::ok(None))
|
||||
)
|
||||
}
|
||||
)
|
||||
.and_then(move |(maybe_code, maybe_content_type, maybe_redirect)| {
|
||||
if let Some(redirect) = maybe_redirect {
|
||||
|
|
@ -140,131 +126,74 @@ pub fn fetch_string_once(
|
|||
}
|
||||
|
||||
#[cfg(test)]
|
||||
/// Asynchronously fetchs the given HTTP URL. Returns (content, media_type).
|
||||
pub fn fetch_string(
|
||||
module_name: &str,
|
||||
) -> impl Future<Item = (String, String), Error = ErrBox> {
|
||||
let url = module_name.parse::<Uri>().unwrap();
|
||||
let client = get_client();
|
||||
// TODO(kevinkassimo): consider set a max redirection counter
|
||||
// to avoid bouncing between 2 or more urls
|
||||
loop_fn((client, url), |(client, url)| {
|
||||
client
|
||||
.get(url.clone())
|
||||
.map_err(ErrBox::from)
|
||||
.and_then(move |response| {
|
||||
if response.status().is_redirection() {
|
||||
let location_string = response
|
||||
.headers()
|
||||
.get("location")
|
||||
.expect("url redirection should provide 'location' header")
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_string();
|
||||
debug!("Redirecting to {}...", &location_string);
|
||||
let new_url = resolve_uri_from_location(&url, &location_string);
|
||||
return Ok(Loop::Continue((client, new_url)));
|
||||
}
|
||||
if !response.status().is_success() {
|
||||
return Err(
|
||||
DenoError::new(
|
||||
deno_error::ErrorKind::NotFound,
|
||||
"module not found".to_string(),
|
||||
)
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
Ok(Loop::Break(response))
|
||||
})
|
||||
})
|
||||
.and_then(|response| {
|
||||
let content_type = response
|
||||
.headers()
|
||||
.get(CONTENT_TYPE)
|
||||
.map(|content_type| content_type.to_str().unwrap().to_string());
|
||||
let body = response
|
||||
.into_body()
|
||||
.concat2()
|
||||
.map(|body| String::from_utf8(body.to_vec()).unwrap())
|
||||
.map_err(ErrBox::from);
|
||||
body.join(future::ok(content_type))
|
||||
})
|
||||
.and_then(|(body_string, maybe_content_type)| {
|
||||
future::ok((body_string, maybe_content_type.unwrap()))
|
||||
})
|
||||
}
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::tokio_util;
|
||||
|
||||
#[test]
|
||||
fn test_fetch_sync_string() {
|
||||
// Relies on external http server. See tools/http_server.py
|
||||
tokio_util::init(|| {
|
||||
let (p, m) =
|
||||
fetch_sync_string("http://127.0.0.1:4545/package.json").unwrap();
|
||||
println!("package.json len {}", p.len());
|
||||
assert!(p.len() > 1);
|
||||
assert!(m == "application/json")
|
||||
});
|
||||
}
|
||||
pub fn fetch_string_once_sync(url: &Url) -> Result<FetchOnceResult, ErrBox> {
|
||||
tokio_util::block_on(fetch_string_once(url))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_fetch_string() {
|
||||
// Relies on external http server. See tools/http_server.py
|
||||
tokio_util::init(|| {
|
||||
let (p, m) = fetch_string("http://127.0.0.1:4545/package.json")
|
||||
.wait()
|
||||
.unwrap();
|
||||
println!("package.json len {}", p.len());
|
||||
assert!(p.len() > 1);
|
||||
assert!(m == "application/json")
|
||||
});
|
||||
}
|
||||
#[test]
|
||||
fn test_fetch_sync_string() {
|
||||
// Relies on external http server. See tools/http_server.py
|
||||
let url = Url::parse("http://127.0.0.1:4545/package.json").unwrap();
|
||||
tokio_util::init(|| match fetch_string_once_sync(&url).unwrap() {
|
||||
FetchOnceResult::Code(code, maybe_content_type) => {
|
||||
assert!(!code.is_empty());
|
||||
assert_eq!(maybe_content_type, Some("application/json".to_string()));
|
||||
}
|
||||
_ => unreachable!(),
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_fetch_sync_string_with_redirect() {
|
||||
// Relies on external http server. See tools/http_server.py
|
||||
tokio_util::init(|| {
|
||||
let (p, m) =
|
||||
fetch_sync_string("http://127.0.0.1:4546/package.json").unwrap();
|
||||
println!("package.json len {}", p.len());
|
||||
assert!(p.len() > 1);
|
||||
assert!(m == "application/json")
|
||||
});
|
||||
}
|
||||
#[test]
|
||||
fn test_fetch_string_once_with_redirect() {
|
||||
// Relies on external http server. See tools/http_server.py
|
||||
let url = Url::parse("http://127.0.0.1:4546/package.json").unwrap();
|
||||
// Dns resolver substitutes `127.0.0.1` with `localhost`
|
||||
let target_url = Url::parse("http://localhost:4545/package.json").unwrap();
|
||||
tokio_util::init(|| {
|
||||
let result = fetch_string_once_sync(&url).unwrap();
|
||||
assert_eq!(result, FetchOnceResult::Redirect(target_url));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_uri_from_location_full_1() {
|
||||
let url = "http://deno.land".parse::<Uri>().unwrap();
|
||||
let new_uri = resolve_uri_from_location(&url, "http://golang.org");
|
||||
assert_eq!(new_uri.host().unwrap(), "golang.org");
|
||||
}
|
||||
#[test]
|
||||
fn test_resolve_url_from_location_full_1() {
|
||||
let url = "http://deno.land".parse::<Url>().unwrap();
|
||||
let new_uri = resolve_url_from_location(&url, "http://golang.org");
|
||||
assert_eq!(new_uri.host_str().unwrap(), "golang.org");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_uri_from_location_full_2() {
|
||||
let url = "https://deno.land".parse::<Uri>().unwrap();
|
||||
let new_uri = resolve_uri_from_location(&url, "https://golang.org");
|
||||
assert_eq!(new_uri.host().unwrap(), "golang.org");
|
||||
}
|
||||
#[test]
|
||||
fn test_resolve_url_from_location_full_2() {
|
||||
let url = "https://deno.land".parse::<Url>().unwrap();
|
||||
let new_uri = resolve_url_from_location(&url, "https://golang.org");
|
||||
assert_eq!(new_uri.host_str().unwrap(), "golang.org");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_uri_from_location_relative_1() {
|
||||
let url = "http://deno.land/x".parse::<Uri>().unwrap();
|
||||
let new_uri = resolve_uri_from_location(&url, "//rust-lang.org/en-US");
|
||||
assert_eq!(new_uri.host().unwrap(), "rust-lang.org");
|
||||
assert_eq!(new_uri.path(), "/en-US");
|
||||
}
|
||||
#[test]
|
||||
fn test_resolve_url_from_location_relative_1() {
|
||||
let url = "http://deno.land/x".parse::<Url>().unwrap();
|
||||
let new_uri = resolve_url_from_location(&url, "//rust-lang.org/en-US");
|
||||
assert_eq!(new_uri.host_str().unwrap(), "rust-lang.org");
|
||||
assert_eq!(new_uri.path(), "/en-US");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_uri_from_location_relative_2() {
|
||||
let url = "http://deno.land/x".parse::<Uri>().unwrap();
|
||||
let new_uri = resolve_uri_from_location(&url, "/y");
|
||||
assert_eq!(new_uri.host().unwrap(), "deno.land");
|
||||
assert_eq!(new_uri.path(), "/y");
|
||||
}
|
||||
#[test]
|
||||
fn test_resolve_url_from_location_relative_2() {
|
||||
let url = "http://deno.land/x".parse::<Url>().unwrap();
|
||||
let new_uri = resolve_url_from_location(&url, "/y");
|
||||
assert_eq!(new_uri.host_str().unwrap(), "deno.land");
|
||||
assert_eq!(new_uri.path(), "/y");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_uri_from_location_relative_3() {
|
||||
let url = "http://deno.land/x".parse::<Uri>().unwrap();
|
||||
let new_uri = resolve_uri_from_location(&url, "z");
|
||||
assert_eq!(new_uri.host().unwrap(), "deno.land");
|
||||
assert_eq!(new_uri.path(), "/z");
|
||||
#[test]
|
||||
fn test_resolve_url_from_location_relative_3() {
|
||||
let url = "http://deno.land/x".parse::<Url>().unwrap();
|
||||
let new_uri = resolve_url_from_location(&url, "z");
|
||||
assert_eq!(new_uri.host_str().unwrap(), "deno.land");
|
||||
assert_eq!(new_uri.path(), "/z");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue