Merge branch 'master' into edition2021

Cargo.toml

@@ -70,11 +70,11 @@ http3 = ["rustls-tls-manual-roots", "h3", "h3-quinn", "quinn", "futures-channel"
 # Don't rely on these whatsoever. They may disappear at anytime.
 
 # Enables common types used for TLS. Useless on its own.
-__tls = []
+__tls = ["dep:rustls-pemfile"]
 
 # Enables common rustls code.
 # Equivalent to rustls-tls-manual-roots but shorter :)
-__rustls = ["hyper-rustls", "tokio-rustls", "rustls", "__tls", "rustls-pemfile"]
+__rustls = ["hyper-rustls", "tokio-rustls", "rustls", "__tls"]
 
 # When enabled, disable using the cached SYS_PROXIES.
 __internal_proxy_sys_no_cache = []
@@ -89,6 +89,7 @@ serde_urlencoded = "0.7.1"
 tower-service = "0.3"
 futures-core = { version = "0.3.0", default-features = false }
 futures-util = { version = "0.3.0", default-features = false }
+sync_wrapper = "0.1.2"
 
 # Optional deps...
 
@@ -101,7 +102,7 @@ mime_guess = { version = "2.0", default-features = false, optional = true }
 encoding_rs = "0.8"
 http-body = "0.4.0"
 hyper = { version = "0.14.21", default-features = false, features = ["tcp", "http1", "http2", "client", "runtime"] }
-h2 = "0.3.10"
+h2 = "0.3.14"
 once_cell = "1"
 log = "0.4"
 mime = "0.3.16"
@@ -111,6 +112,7 @@ pin-project-lite = "0.2.0"
 ipnet = "2.3"
 
 # Optional deps...
+rustls-pemfile = { version = "1.0", optional = true }
 
 ## default-tls
 hyper-tls = { version = "0.5", optional = true }
@@ -123,11 +125,10 @@ rustls = { version = "0.21.6", features = ["dangerous_configuration"], optional
 tokio-rustls = { version = "0.24", optional = true }
 webpki-roots = { version = "0.25", optional = true }
 rustls-native-certs = { version = "0.6", optional = true }
-rustls-pemfile = { version = "1.0", optional = true }
 
 ## cookies
-cookie_crate = { version = "0.16", package = "cookie", optional = true }
-cookie_store = { version = "0.16", optional = true }
+cookie_crate = { version = "0.17.0", package = "cookie", optional = true }
+cookie_store = { version = "0.20.0", optional = true }
 
 ## compression
 async-compression = { version = "0.4.0", default-features = false, features = ["tokio"], optional = true }
@@ -147,13 +148,14 @@ futures-channel = { version = "0.3", optional = true }
 
 
 [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
-env_logger = "0.8"
+env_logger = "0.10"
 hyper = { version = "0.14", default-features = false, features = ["tcp", "stream", "http1", "http2", "client", "server", "runtime"] }
 serde = { version = "1.0", features = ["derive"] }
 libflate = "1.0"
 brotli_crate = { package = "brotli", version = "3.3.0" }
 doc-comment = "0.3"
 tokio = { version = "1.0", default-features = false, features = ["macros", "rt-multi-thread"] }
+futures-util = { version = "0.3.0", default-features = false, features = ["std", "alloc"] }
 
 [target.'cfg(windows)'.dependencies]
 winreg = "0.50.0"
@@ -168,7 +170,7 @@ js-sys = "0.3.45"
 serde_json = "1.0"
 wasm-bindgen = "0.2.68"
 wasm-bindgen-futures = "0.4.18"
-wasm-streams = { version = "0.3", optional = true }
+wasm-streams = { version = "0.4", optional = true }
 
 [target.'cfg(target_arch = "wasm32")'.dependencies.web-sys]
 version = "0.3.25"

src/async_impl/body.rs

@@ -7,6 +7,7 @@ use bytes::Bytes;
 use futures_core::Stream;
 use http_body::Body as HttpBody;
 use pin_project_lite::pin_project;
+use sync_wrapper::SyncWrapper;
 #[cfg(feature = "stream")]
 use tokio::fs::File;
 use tokio::time::Sleep;
@@ -38,7 +39,7 @@ enum Inner {
 pin_project! {
     struct WrapStream<S> {
         #[pin]
-        inner: S,
+        inner: SyncWrapper<S>,
     }
 }
 
@@ -82,7 +83,7 @@ impl Body {
     #[cfg_attr(docsrs, doc(cfg(feature = "stream")))]
     pub fn wrap_stream<S>(stream: S) -> Body
     where
-        S: futures_core::stream::TryStream + Send + Sync + 'static,
+        S: futures_core::stream::TryStream + Send + 'static,
         S::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
         Bytes: From<S::Ok>,
     {
@@ -91,14 +92,14 @@ impl Body {
 
     pub(crate) fn stream<S>(stream: S) -> Body
     where
-        S: futures_core::stream::TryStream + Send + Sync + 'static,
+        S: futures_core::stream::TryStream + Send + 'static,
         S::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
         Bytes: From<S::Ok>,
     {
         use futures_util::TryStreamExt;
 
         let body = Box::pin(WrapStream {
-            inner: stream.map_ok(Bytes::from).map_err(Into::into),
+            inner: SyncWrapper::new(stream.map_ok(Bytes::from).map_err(Into::into)),
         });
         Body {
             inner: Inner::Streaming {
@@ -312,7 +313,7 @@ where
         self: Pin<&mut Self>,
         cx: &mut Context,
     ) -> Poll<Option<Result<Self::Data, Self::Error>>> {
-        let item = futures_core::ready!(self.project().inner.poll_next(cx)?);
+        let item = futures_core::ready!(self.project().inner.get_pin_mut().poll_next(cx)?);
 
         Poll::Ready(item.map(|val| Ok(val.into())))
     }

src/async_impl/client.rs

@@ -37,7 +37,7 @@ use crate::cookie;
 use crate::dns::trust_dns::TrustDnsResolver;
 use crate::dns::{gai::GaiResolver, DnsResolverWithOverrides, DynResolver, Resolve};
 use crate::error;
-use crate::into_url::{expect_uri, try_uri};
+use crate::into_url::try_uri;
 use crate::redirect::{self, remove_sensitive_headers};
 #[cfg(feature = "__tls")]
 use crate::tls::{self, TlsBackend};
@@ -786,7 +786,11 @@ impl ClientBuilder {
     /// Cookies received in responses will be preserved and included in
     /// additional requests.
     ///
-    /// By default, no cookie store is used.
+    /// By default, no cookie store is used. Enabling the cookie store
+    /// with `cookie_store(true)` will set the store to a default implementation.
+    /// It is **not** necessary to call [cookie_store(true)](crate::ClientBuilder::cookie_store) if [cookie_provider(my_cookie_store)](crate::ClientBuilder::cookie_provider)
+    /// is used; calling [cookie_store(true)](crate::ClientBuilder::cookie_store) _after_ [cookie_provider(my_cookie_store)](crate::ClientBuilder::cookie_provider) will result
+    /// in the provided `my_cookie_store` being **overridden** with a default implementation.
     ///
     /// # Optional
     ///
@@ -807,7 +811,10 @@ impl ClientBuilder {
     /// Cookies received in responses will be passed to this store, and
     /// additional requests will query this store for cookies.
     ///
-    /// By default, no cookie store is used.
+    /// By default, no cookie store is used. It is **not** necessary to also call
+    /// [cookie_store(true)](crate::ClientBuilder::cookie_store) if [cookie_provider(my_cookie_store)](crate::ClientBuilder::cookie_provider) is used; calling
+    /// [cookie_store(true)](crate::ClientBuilder::cookie_store) _after_ [cookie_provider(my_cookie_store)](crate::ClientBuilder::cookie_provider) will result
+    /// in the provided `my_cookie_store` being **overridden** with a default implementation.
     ///
     /// # Optional
     ///
@@ -1803,7 +1810,10 @@ impl Client {
             }
         }
 
-        let uri = expect_uri(&url);
+        let uri = match try_uri(&url) {
+            Ok(uri) => uri,
+            _ => return Pending::new_err(error::url_invalid_uri(url)),
+        };
 
         let (reusable, body) = match body {
             Some(body) => {
@@ -2171,7 +2181,8 @@ impl PendingRequest {
         }
         self.retry_count += 1;
 
-        let uri = expect_uri(&self.url);
+        // If it parsed once, it should parse again
+        let uri = try_uri(&self.url).expect("URL was already validated as URI");
 
         *self.as_mut().in_flight().get_mut() = match *self.as_mut().in_flight().as_ref() {
             #[cfg(feature = "http3")]
@@ -2218,9 +2229,16 @@ fn is_retryable_error(err: &(dyn std::error::Error + 'static)) -> bool {
     if let Some(cause) = err.source() {
         if let Some(err) = cause.downcast_ref::<h2::Error>() {
             // They sent us a graceful shutdown, try with a new connection!
-            return err.is_go_away()
-                && err.is_remote()
-                && err.reason() == Some(h2::Reason::NO_ERROR);
+            if err.is_go_away() && err.is_remote() && err.reason() == Some(h2::Reason::NO_ERROR) {
+                return true;
+            }
+
+            // REFUSED_STREAM was sent from the server, which is safe to retry.
+            // https://www.rfc-editor.org/rfc/rfc9113.html#section-8.7-3.2
+            if err.is_reset() && err.is_remote() && err.reason() == Some(h2::Reason::REFUSED_STREAM)
+            {
+                return true;
+            }
         }
     }
     false
@@ -2344,7 +2362,7 @@ impl Future for PendingRequest {
                     //
                     // If not, just log it and skip the redirect.
                     let loc = loc.and_then(|url| {
-                        if try_uri(&url).is_some() {
+                        if try_uri(&url).is_ok() {
                             Some(url)
                         } else {
                             None
@@ -2389,7 +2407,7 @@ impl Future for PendingRequest {
                                 std::mem::replace(self.as_mut().headers(), HeaderMap::new());
 
                             remove_sensitive_headers(&mut headers, &self.url, &self.urls);
-                            let uri = expect_uri(&self.url);
+                            let uri = try_uri(&self.url)?;
                             let body = match self.body {
                                 Some(Some(ref body)) => Body::reusable(body.clone()),
                                 _ => Body::empty(),
@@ -2489,7 +2507,7 @@ fn add_cookie_header(headers: &mut HeaderMap, cookie_store: &dyn cookie::CookieS
 #[cfg(test)]
 mod tests {
     #[tokio::test]
-    async fn execute_request_rejects_invald_urls() {
+    async fn execute_request_rejects_invalid_urls() {
         let url_str = "hxxps://www.rust-lang.org/";
         let url = url::Url::parse(url_str).unwrap();
         let result = crate::get(url.clone()).await;
@@ -2499,4 +2517,17 @@ mod tests {
         assert!(err.is_builder());
         assert_eq!(url_str, err.url().unwrap().as_str());
     }
+
+    /// https://github.com/seanmonstar/reqwest/issues/668
+    #[tokio::test]
+    async fn execute_request_rejects_invalid_hostname() {
+        let url_str = "https://{{hostname}}/";
+        let url = url::Url::parse(url_str).unwrap();
+        let result = crate::get(url.clone()).await;
+
+        assert!(result.is_err());
+        let err = result.err().unwrap();
+        assert!(err.is_builder());
+        assert_eq!(url_str, err.url().unwrap().as_str());
+    }
 }

src/blocking/client.rs

@@ -478,6 +478,14 @@ impl ClientBuilder {
         self.with_inner(|inner| inner.http2_max_frame_size(sz))
     }
 
+    /// This requires the optional `http3` feature to be
+    /// enabled.
+    #[cfg(feature = "http3")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "http3")))]
+    pub fn http3_prior_knowledge(self) -> ClientBuilder {
+        self.with_inner(|inner| inner.http3_prior_knowledge())
+    }
+
     // TCP options
 
     /// Set whether sockets have `TCP_NODELAY` enabled.

src/dns/trust_dns.rs

@@ -2,7 +2,6 @@
 
 use hyper::client::connect::dns::Name;
 use once_cell::sync::OnceCell;
-pub use trust_dns_resolver::config::{ResolverConfig, ResolverOpts};
 use trust_dns_resolver::{lookup_ip::LookupIpIntoIter, system_conf, TokioAsyncResolver};
 
 use std::io;

src/error.rs

@@ -275,6 +275,10 @@ pub(crate) fn url_bad_scheme(url: Url) -> Error {
     Error::new(Kind::Builder, Some(BadScheme)).with_url(url)
 }
 
+pub(crate) fn url_invalid_uri(url: Url) -> Error {
+    Error::new(Kind::Builder, Some("Parsed Url is not a valid Uri")).with_url(url)
+}
+
 if_wasm! {
     pub(crate) fn wasm(js_val: wasm_bindgen::JsValue) -> BoxError {
         format!("{:?}", js_val).into()

src/into_url.rs

@@ -74,14 +74,10 @@ impl<'a> IntoUrlSealed for String {
 }
 
 if_hyper! {
-    pub(crate) fn expect_uri(url: &Url) -> http::Uri {
+    pub(crate) fn try_uri(url: &Url) -> crate::Result<http::Uri> {
         url.as_str()
             .parse()
-            .expect("a parsed Url should always be a valid Uri")
-    }
-
-    pub(crate) fn try_uri(url: &Url) -> Option<http::Uri> {
-        url.as_str().parse().ok()
+            .map_err(|_| crate::error::url_invalid_uri(url.clone()))
     }
 }
 

src/lib.rs

@@ -15,7 +15,7 @@
 //! - Plain bodies, [JSON](#json), [urlencoded](#forms), [multipart]
 //! - Customizable [redirect policy](#redirect-policies)
 //! - HTTP [Proxies](#proxies)
-//! - Uses system-native [TLS](#tls)
+//! - Uses [TLS](#tls) by default
 //! - Cookies
 //!
 //! The [`reqwest::Client`][client] is asynchronous. For applications wishing
@@ -149,17 +149,18 @@
 //!
 //! ## TLS
 //!
-//! By default, a `Client` will make use of system-native transport layer
-//! security to connect to HTTPS destinations. This means schannel on Windows,
-//! Security-Framework on macOS, and OpenSSL on Linux.
+//! A `Client` will use transport layer security (TLS) by default to connect to
+//! HTTPS destinations.
 //!
-//! - Additional X509 certificates can be configured on a `ClientBuilder` with the
-//!   [`Certificate`] type.
+//! - Additional server certificates can be configured on a `ClientBuilder`
+//!   with the [`Certificate`] type.
 //! - Client certificates can be added to a `ClientBuilder` with the
 //!   [`Identity`] type.
 //! - Various parts of TLS can also be configured or even disabled on the
 //!   `ClientBuilder`.
 //!
+//! See more details in the [`tls`] module.
+//!
 //! ## WASM
 //!
 //! The Client implementation automatically switches to the WASM one when the target_arch is wasm32,
@@ -312,6 +313,9 @@ fn _assert_impls() {
 
     assert_send::<Error>();
     assert_sync::<Error>();
+
+    assert_send::<Body>();
+    assert_sync::<Body>();
 }
 
 if_hyper! {