720 lines
27 KiB
Python
720 lines
27 KiB
Python
|
"""
|
||
|
|
requests.adapters
|
||
|
|
~~~~~~~~~~~~~~~~~
|
||
|
|
|
||
|
|
This module contains the transport adapters that Requests uses to define
|
||
|
|
and maintain connections.
|
||
|
|
"""
|
||
|
|
|
||
|
|
import os.path
|
||
|
|
import socket # noqa: F401
|
||
|
|
import typing
|
||
|
|
import warnings
|
||
|
|
|
||
|
|
from urllib3.exceptions import ClosedPoolError, ConnectTimeoutError
|
||
|
|
from urllib3.exceptions import HTTPError as _HTTPError
|
||
|
|
from urllib3.exceptions import InvalidHeader as _InvalidHeader
|
||
|
|
from urllib3.exceptions import (
|
||
|
|
LocationValueError,
|
||
|
|
MaxRetryError,
|
||
|
|
NewConnectionError,
|
||
|
|
ProtocolError,
|
||
|
|
)
|
||
|
|
from urllib3.exceptions import ProxyError as _ProxyError
|
||
|
|
from urllib3.exceptions import ReadTimeoutError, ResponseError
|
||
|
|
from urllib3.exceptions import SSLError as _SSLError
|
||
|
|
from urllib3.poolmanager import PoolManager, proxy_from_url
|
||
|
|
from urllib3.util import Timeout as TimeoutSauce
|
||
|
|
from urllib3.util import parse_url
|
||
|
|
from urllib3.util.retry import Retry
|
||
|
|
from urllib3.util.ssl_ import create_urllib3_context
|
||
|
|
|
||
|
|
from .auth import _basic_auth_str
|
||
|
|
from .compat import basestring, urlparse
|
||
|
|
from .cookies import extract_cookies_to_jar
|
||
|
|
from .exceptions import (
|
||
|
|
ConnectionError,
|
||
|
|
ConnectTimeout,
|
||
|
|
InvalidHeader,
|
||
|
|
InvalidProxyURL,
|
||
|
|
InvalidSchema,
|
||
|
|
InvalidURL,
|
||
|
|
ProxyError,
|
||
|
|
ReadTimeout,
|
||
|
|
RetryError,
|
||
|
|
SSLError,
|
||
|
|
)
|
||
|
|
from .models import Response
|
||
|
|
from .structures import CaseInsensitiveDict
|
||
|
|
from .utils import (
|
||
|
|
DEFAULT_CA_BUNDLE_PATH,
|
||
|
|
extract_zipped_paths,
|
||
|
|
get_auth_from_url,
|
||
|
|
get_encoding_from_headers,
|
||
|
|
prepend_scheme_if_needed,
|
||
|
|
select_proxy,
|
||
|
|
urldefragauth,
|
||
|
|
)
|
||
|
|
|
||
|
|
try:
|
||
|
|
from urllib3.contrib.socks import SOCKSProxyManager
|
||
|
|
except ImportError:
|
||
|
|
|
||
|
|
def SOCKSProxyManager(*args, **kwargs):
|
||
|
|
raise InvalidSchema("Missing dependencies for SOCKS support.")
|
||
|
|
|
||
|
|
|
||
|
|
if typing.TYPE_CHECKING:
|
||
|
|
from .models import PreparedRequest
|
||
|
|
|
||
|
|
|
||
|
|
DEFAULT_POOLBLOCK = False
|
||
|
|
DEFAULT_POOLSIZE = 10
|
||
|
|
DEFAULT_RETRIES = 0
|
||
|
|
DEFAULT_POOL_TIMEOUT = None
|
||
|
|
|
||
|
|
|
||
|
|
try:
|
||
|
|
import ssl # noqa: F401
|
||
|
|
|
||
|
|
_preloaded_ssl_context = create_urllib3_context()
|
||
|
|
_preloaded_ssl_context.load_verify_locations(
|
||
|
|
extract_zipped_paths(DEFAULT_CA_BUNDLE_PATH)
|
||
|
|
)
|
||
|
|
except ImportError:
|
||
|
|
# Bypass default SSLContext creation when Python
|
||
|
|
# interpreter isn't built with the ssl module.
|
||
|
|
_preloaded_ssl_context = None
|
||
|
|
|
||
|
|
|
||
|
|
def _urllib3_request_context(
|
||
|
|
request: "PreparedRequest",
|
||
|
|
verify: "bool | str | None",
|
||
|
|
client_cert: "typing.Tuple[str, str] | str | None",
|
||
|
|
poolmanager: "PoolManager",
|
||
|
|
) -> "(typing.Dict[str, typing.Any], typing.Dict[str, typing.Any])":
|
||
|
|
host_params = {}
|
||
|
|
pool_kwargs = {}
|
||
|
|
parsed_request_url = urlparse(request.url)
|
||
|
|
scheme = parsed_request_url.scheme.lower()
|
||
|
|
port = parsed_request_url.port
|
||
|
|
|
||
|
|
# Determine if we have and should use our default SSLContext
|
||
|
|
# to optimize performance on standard requests.
|
||
|
|
poolmanager_kwargs = getattr(poolmanager, "connection_pool_kw", {})
|
||
|
|
has_poolmanager_ssl_context = poolmanager_kwargs.get("ssl_context")
|
||
|
|
should_use_default_ssl_context = (
|
||
|
|
_preloaded_ssl_context is not None and not has_poolmanager_ssl_context
|
||
|
|
)
|
||
|
|
|
||
|
|
cert_reqs = "CERT_REQUIRED"
|
||
|
|
if verify is False:
|
||
|
|
cert_reqs = "CERT_NONE"
|
||
|
|
elif verify is True and should_use_default_ssl_context:
|
||
|
|
pool_kwargs["ssl_context"] = _preloaded_ssl_context
|
||
|
|
elif isinstance(verify, str):
|
||
|
|
if not os.path.isdir(verify):
|
||
|
|
pool_kwargs["ca_certs"] = verify
|
||
|
|
else:
|
||
|
|
pool_kwargs["ca_cert_dir"] = verify
|
||
|
|
pool_kwargs["cert_reqs"] = cert_reqs
|
||
|
|
if client_cert is not None:
|
||
|
|
if isinstance(client_cert, tuple) and len(client_cert) == 2:
|
||
|
|
pool_kwargs["cert_file"] = client_cert[0]
|
||
|
|
pool_kwargs["key_file"] = client_cert[1]
|
||
|
|
else:
|
||
|
|
# According to our docs, we allow users to specify just the client
|
||
|
|
# cert path
|
||
|
|
pool_kwargs["cert_file"] = client_cert
|
||
|
|
host_params = {
|
||
|
|
"scheme": scheme,
|
||
|
|
"host": parsed_request_url.hostname,
|
||
|
|
"port": port,
|
||
|
|
}
|
||
|
|
return host_params, pool_kwargs
|
||
|
|
|
||
|
|
|
||
|
|
class BaseAdapter:
|
||
|
|
"""The Base Transport Adapter"""
|
||
|
|
|
||
|
|
def __init__(self):
|
||
|
|
super().__init__()
|
||
|
|
|
||
|
|
def send(
|
||
|
|
self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None
|
||
|
|
):
|
||
|
|
"""Sends PreparedRequest object. Returns Response object.
|
||
|
|
|
||
|
|
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
|
||
|
|
:param stream: (optional) Whether to stream the request content.
|
||
|
|
:param timeout: (optional) How long to wait for the server to send
|
||
|
|
data before giving up, as a float, or a :ref:`(connect timeout,
|
||
|
|
read timeout) <timeouts>` tuple.
|
||
|
|
:type timeout: float or tuple
|
||
|
|
:param verify: (optional) Either a boolean, in which case it controls whether we verify
|
||
|
|
the server's TLS certificate, or a string, in which case it must be a path
|
||
|
|
to a CA bundle to use
|
||
|
|
:param cert: (optional) Any user-provided SSL certificate to be trusted.
|
||
|
|
:param proxies: (optional) The proxies dictionary to apply to the request.
|
||
|
|
"""
|
||
|
|
raise NotImplementedError
|
||
|
|
|
||
|
|
def close(self):
|
||
|
|
"""Cleans up adapter specific items."""
|
||
|
|
raise NotImplementedError
|
||
|
|
|
||
|
|
|
||
|
|
class HTTPAdapter(BaseAdapter):
|
||
|
|
"""The built-in HTTP Adapter for urllib3.
|
||
|
|
|
||
|
|
Provides a general-case interface for Requests sessions to contact HTTP and
|
||
|
|
HTTPS urls by implementing the Transport Adapter interface. This class will
|
||
|
|
usually be created by the :class:`Session <Session>` class under the
|
||
|
|
covers.
|
||
|
|
|
||
|
|
:param pool_connections: The number of urllib3 connection pools to cache.
|
||
|
|
:param pool_maxsize: The maximum number of connections to save in the pool.
|
||
|
|
:param max_retries: The maximum number of retries each connection
|
||
|
|
should attempt. Note, this applies only to failed DNS lookups, socket
|
||
|
|
connections and connection timeouts, never to requests where data has
|
||
|
|
made it to the server. By default, Requests does not retry failed
|
||
|
|
connections. If you need granular control over the conditions under
|
||
|
|
which we retry a request, import urllib3's ``Retry`` class and pass
|
||
|
|
that instead.
|
||
|
|
:param pool_block: Whether the connection pool should block for connections.
|
||
|
|
|
||
|
|
Usage::
|
||
|
|
|
||
|
|
>>> import requests
|
||
|
|
>>> s = requests.Session()
|
||
|
|
>>> a = requests.adapters.HTTPAdapter(max_retries=3)
|
||
|
|
>>> s.mount('http://', a)
|
||
|
|
"""
|
||
|
|
|
||
|
|
__attrs__ = [
|
||
|
|
"max_retries",
|
||
|
|
"config",
|
||
|
|
"_pool_connections",
|
||
|
|
"_pool_maxsize",
|
||
|
|
"_pool_block",
|
||
|
|
]
|
||
|
|
|
||
|
|
def __init__(
|
||
|
|
self,
|
||
|
|
pool_connections=DEFAULT_POOLSIZE,
|
||
|
|
pool_maxsize=DEFAULT_POOLSIZE,
|
||
|
|
max_retries=DEFAULT_RETRIES,
|
||
|
|
pool_block=DEFAULT_POOLBLOCK,
|
||
|
|
):
|
||
|
|
if max_retries == DEFAULT_RETRIES:
|
||
|
|
self.max_retries = Retry(0, read=False)
|
||
|
|
else:
|
||
|
|
self.max_retries = Retry.from_int(max_retries)
|
||
|
|
self.config = {}
|
||
|
|
self.proxy_manager = {}
|
||
|
|
|
||
|
|
super().__init__()
|
||
|
|
|
||
|
|
self._pool_connections = pool_connections
|
||
|
|
self._pool_maxsize = pool_maxsize
|
||
|
|
self._pool_block = pool_block
|
||
|
|
|
||
|
|
self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block)
|
||
|
|
|
||
|
|
def __getstate__(self):
|
||
|
|
return {attr: getattr(self, attr, None) for attr in self.__attrs__}
|
||
|
|
|
||
|
|
def __setstate__(self, state):
|
||
|
|
# Can't handle by adding 'proxy_manager' to self.__attrs__ because
|
||
|
|
# self.poolmanager uses a lambda function, which isn't pickleable.
|
||
|
|
self.proxy_manager = {}
|
||
|
|
self.config = {}
|
||
|
|
|
||
|
|
for attr, value in state.items():
|
||
|
|
setattr(self, attr, value)
|
||
|
|
|
||
|
|
self.init_poolmanager(
|
||
|
|
self._pool_connections, self._pool_maxsize, block=self._pool_block
|
||
|
|
)
|
||
|
|
|
||
|
|
def init_poolmanager(
|
||
|
|
self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs
|
||
|
|
):
|
||
|
|
"""Initializes a urllib3 PoolManager.
|
||
|
|
|
||
|
|
This method should not be called from user code, and is only
|
||
|
|
exposed for use when subclassing the
|
||
|
|
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
|
||
|
|
|
||
|
|
:param connections: The number of urllib3 connection pools to cache.
|
||
|
|
:param maxsize: The maximum number of connections to save in the pool.
|
||
|
|
:param block: Block when no free connections are available.
|
||
|
|
:param pool_kwargs: Extra keyword arguments used to initialize the Pool Manager.
|
||
|
|
"""
|
||
|
|
# save these values for pickling
|
||
|
|
self._pool_connections = connections
|
||
|
|
self._pool_maxsize = maxsize
|
||
|
|
self._pool_block = block
|
||
|
|
|
||
|
|
self.poolmanager = PoolManager(
|
||
|
|
num_pools=connections,
|
||
|
|
maxsize=maxsize,
|
||
|
|
block=block,
|
||
|
|
**pool_kwargs,
|
||
|
|
)
|
||
|
|
|
||
|
|
def proxy_manager_for(self, proxy, **proxy_kwargs):
|
||
|
|
"""Return urllib3 ProxyManager for the given proxy.
|
||
|
|
|
||
|
|
This method should not be called from user code, and is only
|
||
|
|
exposed for use when subclassing the
|
||
|
|
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
|
||
|
|
|
||
|
|
:param proxy: The proxy to return a urllib3 ProxyManager for.
|
||
|
|
:param proxy_kwargs: Extra keyword arguments used to configure the Proxy Manager.
|
||
|
|
:returns: ProxyManager
|
||
|
|
:rtype: urllib3.ProxyManager
|
||
|
|
"""
|
||
|
|
if proxy in self.proxy_manager:
|
||
|
|
manager = self.proxy_manager[proxy]
|
||
|
|
elif proxy.lower().startswith("socks"):
|
||
|
|
username, password = get_auth_from_url(proxy)
|
||
|
|
manager = self.proxy_manager[proxy] = SOCKSProxyManager(
|
||
|
|
proxy,
|
||
|
|
username=username,
|
||
|
|
password=password,
|
||
|
|
num_pools=self._pool_connections,
|
||
|
|
maxsize=self._pool_maxsize,
|
||
|
|
block=self._pool_block,
|
||
|
|
**proxy_kwargs,
|
||
|
|
)
|
||
|
|
else:
|
||
|
|
proxy_headers = self.proxy_headers(proxy)
|
||
|
|
manager = self.proxy_manager[proxy] = proxy_from_url(
|
||
|
|
proxy,
|
||
|
|
proxy_headers=proxy_headers,
|
||
|
|
num_pools=self._pool_connections,
|
||
|
|
maxsize=self._pool_maxsize,
|
||
|
|
block=self._pool_block,
|
||
|
|
**proxy_kwargs,
|
||
|
|
)
|
||
|
|
|
||
|
|
return manager
|
||
|
|
|
||
|
|
def cert_verify(self, conn, url, verify, cert):
|
||
|
|
"""Verify a SSL certificate. This method should not be called from user
|
||
|
|
code, and is only exposed for use when subclassing the
|
||
|
|
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
|
||
|
|
|
||
|
|
:param conn: The urllib3 connection object associated with the cert.
|
||
|
|
:param url: The requested URL.
|
||
|
|
:param verify: Either a boolean, in which case it controls whether we verify
|
||
|
|
the server's TLS certificate, or a string, in which case it must be a path
|
||
|
|
to a CA bundle to use
|
||
|
|
:param cert: The SSL certificate to verify.
|
||
|
|
"""
|
||
|
|
if url.lower().startswith("https") and verify:
|
||
|
|
conn.cert_reqs = "CERT_REQUIRED"
|
||
|
|
|
||
|
|
# Only load the CA certificates if 'verify' is a string indicating the CA bundle to use.
|
||
|
|
# Otherwise, if verify is a boolean, we don't load anything since
|
||
|
|
# the connection will be using a context with the default certificates already loaded,
|
||
|
|
# and this avoids a call to the slow load_verify_locations()
|
||
|
|
if verify is not True:
|
||
|
|
# `verify` must be a str with a path then
|
||
|
|
cert_loc = verify
|
||
|
|
|
||
|
|
if not os.path.exists(cert_loc):
|
||
|
|
raise OSError(
|
||
|
|
f"Could not find a suitable TLS CA certificate bundle, "
|
||
|
|
f"invalid path: {cert_loc}"
|
||
|
|
)
|
||
|
|
|
||
|
|
if not os.path.isdir(cert_loc):
|
||
|
|
conn.ca_certs = cert_loc
|
||
|
|
else:
|
||
|
|
conn.ca_cert_dir = cert_loc
|
||
|
|
else:
|
||
|
|
conn.cert_reqs = "CERT_NONE"
|
||
|
|
conn.ca_certs = None
|
||
|
|
conn.ca_cert_dir = None
|
||
|
|
|
||
|
|
if cert:
|
||
|
|
if not isinstance(cert, basestring):
|
||
|
|
conn.cert_file = cert[0]
|
||
|
|
conn.key_file = cert[1]
|
||
|
|
else:
|
||
|
|
conn.cert_file = cert
|
||
|
|
conn.key_file = None
|
||
|
|
if conn.cert_file and not os.path.exists(conn.cert_file):
|
||
|
|
raise OSError(
|
||
|
|
f"Could not find the TLS certificate file, "
|
||
|
|
f"invalid path: {conn.cert_file}"
|
||
|
|
)
|
||
|
|
if conn.key_file and not os.path.exists(conn.key_file):
|
||
|
|
raise OSError(
|
||
|
|
f"Could not find the TLS key file, invalid path: {conn.key_file}"
|
||
|
|
)
|
||
|
|
|
||
|
|
def build_response(self, req, resp):
|
||
|
|
"""Builds a :class:`Response <requests.Response>` object from a urllib3
|
||
|
|
response. This should not be called from user code, and is only exposed
|
||
|
|
for use when subclassing the
|
||
|
|
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`
|
||
|
|
|
||
|
|
:param req: The :class:`PreparedRequest <PreparedRequest>` used to generate the response.
|
||
|
|
:param resp: The urllib3 response object.
|
||
|
|
:rtype: requests.Response
|
||
|
|
"""
|
||
|
|
response = Response()
|
||
|
|
|
||
|
|
# Fallback to None if there's no status_code, for whatever reason.
|
||
|
|
response.status_code = getattr(resp, "status", None)
|
||
|
|
|
||
|
|
# Make headers case-insensitive.
|
||
|
|
response.headers = CaseInsensitiveDict(getattr(resp, "headers", {}))
|
||
|
|
|
||
|
|
# Set encoding.
|
||
|
|
response.encoding = get_encoding_from_headers(response.headers)
|
||
|
|
response.raw = resp
|
||
|
|
response.reason = response.raw.reason
|
||
|
|
|
||
|
|
if isinstance(req.url, bytes):
|
||
|
|
response.url = req.url.decode("utf-8")
|
||
|
|
else:
|
||
|
|
response.url = req.url
|
||
|
|
|
||
|
|
# Add new cookies from the server.
|
||
|
|
extract_cookies_to_jar(response.cookies, req, resp)
|
||
|
|
|
||
|
|
# Give the Response some context.
|
||
|
|
response.request = req
|
||
|
|
response.connection = self
|
||
|
|
|
||
|
|
return response
|
||
|
|
|
||
|
|
def build_connection_pool_key_attributes(self, request, verify, cert=None):
|
||
|
|
"""Build the PoolKey attributes used by urllib3 to return a connection.
|
||
|
|
|
||
|
|
This looks at the PreparedRequest, the user-specified verify value,
|
||
|
|
and the value of the cert parameter to determine what PoolKey values
|
||
|
|
to use to select a connection from a given urllib3 Connection Pool.
|
||
|
|
|
||
|
|
The SSL related pool key arguments are not consistently set. As of
|
||
|
|
this writing, use the following to determine what keys may be in that
|
||
|
|
dictionary:
|
||
|
|
|
||
|
|
* If ``verify`` is ``True``, ``"ssl_context"`` will be set and will be the
|
||
|
|
default Requests SSL Context
|
||
|
|
* If ``verify`` is ``False``, ``"ssl_context"`` will not be set but
|
||
|
|
``"cert_reqs"`` will be set
|
||
|
|
* If ``verify`` is a string, (i.e., it is a user-specified trust bundle)
|
||
|
|
``"ca_certs"`` will be set if the string is not a directory recognized
|
||
|
|
by :py:func:`os.path.isdir`, otherwise ``"ca_certs_dir"`` will be
|
||
|
|
set.
|
||
|
|
* If ``"cert"`` is specified, ``"cert_file"`` will always be set. If
|
||
|
|
``"cert"`` is a tuple with a second item, ``"key_file"`` will also
|
||
|
|
be present
|
||
|
|
|
||
|
|
To override these settings, one may subclass this class, call this
|
||
|
|
method and use the above logic to change parameters as desired. For
|
||
|
|
example, if one wishes to use a custom :py:class:`ssl.SSLContext` one
|
||
|
|
must both set ``"ssl_context"`` and based on what else they require,
|
||
|
|
alter the other keys to ensure the desired behaviour.
|
||
|
|
|
||
|
|
:param request:
|
||
|
|
The PreparedReqest being sent over the connection.
|
||
|
|
:type request:
|
||
|
|
:class:`~requests.models.PreparedRequest`
|
||
|
|
:param verify:
|
||
|
|
Either a boolean, in which case it controls whether
|
||
|
|
we verify the server's TLS certificate, or a string, in which case it
|
||
|
|
must be a path to a CA bundle to use.
|
||
|
|
:param cert:
|
||
|
|
(optional) Any user-provided SSL certificate for client
|
||
|
|
authentication (a.k.a., mTLS). This may be a string (i.e., just
|
||
|
|
the path to a file which holds both certificate and key) or a
|
||
|
|
tuple of length 2 with the certificate file path and key file
|
||
|
|
path.
|
||
|
|
:returns:
|
||
|
|
A tuple of two dictionaries. The first is the "host parameters"
|
||
|
|
portion of the Pool Key including scheme, hostname, and port. The
|
||
|
|
second is a dictionary of SSLContext related parameters.
|
||
|
|
"""
|
||
|
|
return _urllib3_request_context(request, verify, cert, self.poolmanager)
|
||
|
|
|
||
|
|
def get_connection_with_tls_context(self, request, verify, proxies=None, cert=None):
|
||
|
|
"""Returns a urllib3 connection for the given request and TLS settings.
|
||
|
|
This should not be called from user code, and is only exposed for use
|
||
|
|
when subclassing the :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
|
||
|
|
|
||
|
|
:param request:
|
||
|
|
The :class:`PreparedRequest <PreparedRequest>` object to be sent
|
||
|
|
over the connection.
|
||
|
|
:param verify:
|
||
|
|
Either a boolean, in which case it controls whether we verify the
|
||
|
|
server's TLS certificate, or a string, in which case it must be a
|
||
|
|
path to a CA bundle to use.
|
||
|
|
:param proxies:
|
||
|
|
(optional) The proxies dictionary to apply to the request.
|
||
|
|
:param cert:
|
||
|
|
(optional) Any user-provided SSL certificate to be used for client
|
||
|
|
authentication (a.k.a., mTLS).
|
||
|
|
:rtype:
|
||
|
|
urllib3.ConnectionPool
|
||
|
|
"""
|
||
|
|
proxy = select_proxy(request.url, proxies)
|
||
|
|
try:
|
||
|
|
host_params, pool_kwargs = self.build_connection_pool_key_attributes(
|
||
|
|
request,
|
||
|
|
verify,
|
||
|
|
cert,
|
||
|
|
)
|
||
|
|
except ValueError as e:
|
||
|
|
raise InvalidURL(e, request=request)
|
||
|
|
if proxy:
|
||
|
|
proxy = prepend_scheme_if_needed(proxy, "http")
|
||
|
|
proxy_url = parse_url(proxy)
|
||
|
|
if not proxy_url.host:
|
||
|
|
raise InvalidProxyURL(
|
||
|
|
"Please check proxy URL. It is malformed "
|
||
|
|
"and could be missing the host."
|
||
|
|
)
|
||
|
|
proxy_manager = self.proxy_manager_for(proxy)
|
||
|
|
conn = proxy_manager.connection_from_host(
|
||
|
|
**host_params, pool_kwargs=pool_kwargs
|
||
|
|
)
|
||
|
|
else:
|
||
|
|
# Only scheme should be lower case
|
||
|
|
conn = self.poolmanager.connection_from_host(
|
||
|
|
**host_params, pool_kwargs=pool_kwargs
|
||
|
|
)
|
||
|
|
|
||
|
|
return conn
|
||
|
|
|
||
|
|
def get_connection(self, url, proxies=None):
|
||
|
|
"""DEPRECATED: Users should move to `get_connection_with_tls_context`
|
||
|
|
for all subclasses of HTTPAdapter using Requests>=2.32.2.
|
||
|
|
|
||
|
|
Returns a urllib3 connection for the given URL. This should not be
|
||
|
|
called from user code, and is only exposed for use when subclassing the
|
||
|
|
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
|
||
|
|
|
||
|
|
:param url: The URL to connect to.
|
||
|
|
:param proxies: (optional) A Requests-style dictionary of proxies used on this request.
|
||
|
|
:rtype: urllib3.ConnectionPool
|
||
|
|
"""
|
||
|
|
warnings.warn(
|
||
|
|
(
|
||
|
|
"`get_connection` has been deprecated in favor of "
|
||
|
|
"`get_connection_with_tls_context`. Custom HTTPAdapter subclasses "
|
||
|
|
"will need to migrate for Requests>=2.32.2. Please see "
|
||
|
|
"https://github.com/psf/requests/pull/6710 for more details."
|
||
|
|
),
|
||
|
|
DeprecationWarning,
|
||
|
|
)
|
||
|
|
proxy = select_proxy(url, proxies)
|
||
|
|
|
||
|
|
if proxy:
|
||
|
|
proxy = prepend_scheme_if_needed(proxy, "http")
|
||
|
|
proxy_url = parse_url(proxy)
|
||
|
|
if not proxy_url.host:
|
||
|
|
raise InvalidProxyURL(
|
||
|
|
"Please check proxy URL. It is malformed "
|
||
|
|
"and could be missing the host."
|
||
|
|
)
|
||
|
|
proxy_manager = self.proxy_manager_for(proxy)
|
||
|
|
conn = proxy_manager.connection_from_url(url)
|
||
|
|
else:
|
||
|
|
# Only scheme should be lower case
|
||
|
|
parsed = urlparse(url)
|
||
|
|
url = parsed.geturl()
|
||
|
|
conn = self.poolmanager.connection_from_url(url)
|
||
|
|
|
||
|
|
return conn
|
||
|
|
|
||
|
|
def close(self):
|
||
|
|
"""Disposes of any internal state.
|
||
|
|
|
||
|
|
Currently, this closes the PoolManager and any active ProxyManager,
|
||
|
|
which closes any pooled connections.
|
||
|
|
"""
|
||
|
|
self.poolmanager.clear()
|
||
|
|
for proxy in self.proxy_manager.values():
|
||
|
|
proxy.clear()
|
||
|
|
|
||
|
|
def request_url(self, request, proxies):
|
||
|
|
"""Obtain the url to use when making the final request.
|
||
|
|
|
||
|
|
If the message is being sent through a HTTP proxy, the full URL has to
|
||
|
|
be used. Otherwise, we should only use the path portion of the URL.
|
||
|
|
|
||
|
|
This should not be called from user code, and is only exposed for use
|
||
|
|
when subclassing the
|
||
|
|
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
|
||
|
|
|
||
|
|
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
|
||
|
|
:param proxies: A dictionary of schemes or schemes and hosts to proxy URLs.
|
||
|
|
:rtype: str
|
||
|
|
"""
|
||
|
|
proxy = select_proxy(request.url, proxies)
|
||
|
|
scheme = urlparse(request.url).scheme
|
||
|
|
|
||
|
|
is_proxied_http_request = proxy and scheme != "https"
|
||
|
|
using_socks_proxy = False
|
||
|
|
if proxy:
|
||
|
|
proxy_scheme = urlparse(proxy).scheme.lower()
|
||
|
|
using_socks_proxy = proxy_scheme.startswith("socks")
|
||
|
|
|
||
|
|
url = request.path_url
|
||
|
|
if url.startswith("//"): # Don't confuse urllib3
|
||
|
|
url = f"/{url.lstrip('/')}"
|
||
|
|
|
||
|
|
if is_proxied_http_request and not using_socks_proxy:
|
||
|
|
url = urldefragauth(request.url)
|
||
|
|
|
||
|
|
return url
|
||
|
|
|
||
|
|
def add_headers(self, request, **kwargs):
|
||
|
|
"""Add any headers needed by the connection. As of v2.0 this does
|
||
|
|
nothing by default, but is left for overriding by users that subclass
|
||
|
|
the :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
|
||
|
|
|
||
|
|
This should not be called from user code, and is only exposed for use
|
||
|
|
when subclassing the
|
||
|
|
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
|
||
|
|
|
||
|
|
:param request: The :class:`PreparedRequest <PreparedRequest>` to add headers to.
|
||
|
|
:param kwargs: The keyword arguments from the call to send().
|
||
|
|
"""
|
||
|
|
pass
|
||
|
|
|
||
|
|
def proxy_headers(self, proxy):
|
||
|
|
"""Returns a dictionary of the headers to add to any request sent
|
||
|
|
through a proxy. This works with urllib3 magic to ensure that they are
|
||
|
|
correctly sent to the proxy, rather than in a tunnelled request if
|
||
|
|
CONNECT is being used.
|
||
|
|
|
||
|
|
This should not be called from user code, and is only exposed for use
|
||
|
|
when subclassing the
|
||
|
|
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
|
||
|
|
|
||
|
|
:param proxy: The url of the proxy being used for this request.
|
||
|
|
:rtype: dict
|
||
|
|
"""
|
||
|
|
headers = {}
|
||
|
|
username, password = get_auth_from_url(proxy)
|
||
|
|
|
||
|
|
if username:
|
||
|
|
headers["Proxy-Authorization"] = _basic_auth_str(username, password)
|
||
|
|
|
||
|
|
return headers
|
||
|
|
|
||
|
|
def send(
|
||
|
|
self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None
|
||
|
|
):
|
||
|
|
"""Sends PreparedRequest object. Returns Response object.
|
||
|
|
|
||
|
|
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
|
||
|
|
:param stream: (optional) Whether to stream the request content.
|
||
|
|
:param timeout: (optional) How long to wait for the server to send
|
||
|
|
data before giving up, as a float, or a :ref:`(connect timeout,
|
||
|
|
read timeout) <timeouts>` tuple.
|
||
|
|
:type timeout: float or tuple or urllib3 Timeout object
|
||
|
|
:param verify: (optional) Either a boolean, in which case it controls whether
|
||
|
|
we verify the server's TLS certificate, or a string, in which case it
|
||
|
|
must be a path to a CA bundle to use
|
||
|
|
:param cert: (optional) Any user-provided SSL certificate to be trusted.
|
||
|
|
:param proxies: (optional) The proxies dictionary to apply to the request.
|
||
|
|
:rtype: requests.Response
|
||
|
|
"""
|
||
|
|
|
||
|
|
try:
|
||
|
|
conn = self.get_connection_with_tls_context(
|
||
|
|
request, verify, proxies=proxies, cert=cert
|
||
|
|
)
|
||
|
|
except LocationValueError as e:
|
||
|
|
raise InvalidURL(e, request=request)
|
||
|
|
|
||
|
|
self.cert_verify(conn, request.url, verify, cert)
|
||
|
|
url = self.request_url(request, proxies)
|
||
|
|
self.add_headers(
|
||
|
|
request,
|
||
|
|
stream=stream,
|
||
|
|
timeout=timeout,
|
||
|
|
verify=verify,
|
||
|
|
cert=cert,
|
||
|
|
proxies=proxies,
|
||
|
|
)
|
||
|
|
|
||
|
|
chunked = not (request.body is None or "Content-Length" in request.headers)
|
||
|
|
|
||
|
|
if isinstance(timeout, tuple):
|
||
|
|
try:
|
||
|
|
connect, read = timeout
|
||
|
|
timeout = TimeoutSauce(connect=connect, read=read)
|
||
|
|
except ValueError:
|
||
|
|
raise ValueError(
|
||
|
|
f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, "
|
||
|
|
f"or a single float to set both timeouts to the same value."
|
||
|
|
)
|
||
|
|
elif isinstance(timeout, TimeoutSauce):
|
||
|
|
pass
|
||
|
|
else:
|
||
|
|
timeout = TimeoutSauce(connect=timeout, read=timeout)
|
||
|
|
|
||
|
|
try:
|
||
|
|
resp = conn.urlopen(
|
||
|
|
method=request.method,
|
||
|
|
url=url,
|
||
|
|
body=request.body,
|
||
|
|
headers=request.headers,
|
||
|
|
redirect=False,
|
||
|
|
assert_same_host=False,
|
||
|
|
preload_content=False,
|
||
|
|
decode_content=False,
|
||
|
|
retries=self.max_retries,
|
||
|
|
timeout=timeout,
|
||
|
|
chunked=chunked,
|
||
|
|
)
|
||
|
|
|
||
|
|
except (ProtocolError, OSError) as err:
|
||
|
|
raise ConnectionError(err, request=request)
|
||
|
|
|
||
|
|
except MaxRetryError as e:
|
||
|
|
if isinstance(e.reason, ConnectTimeoutError):
|
||
|
|
# TODO: Remove this in 3.0.0: see #2811
|
||
|
|
if not isinstance(e.reason, NewConnectionError):
|
||
|
|
raise ConnectTimeout(e, request=request)
|
||
|
|
|
||
|
|
if isinstance(e.reason, ResponseError):
|
||
|
|
raise RetryError(e, request=request)
|
||
|
|
|
||
|
|
if isinstance(e.reason, _ProxyError):
|
||
|
|
raise ProxyError(e, request=request)
|
||
|
|
|
||
|
|
if isinstance(e.reason, _SSLError):
|
||
|
|
# This branch is for urllib3 v1.22 and later.
|
||
|
|
raise SSLError(e, request=request)
|
||
|
|
|
||
|
|
raise ConnectionError(e, request=request)
|
||
|
|
|
||
|
|
except ClosedPoolError as e:
|
||
|
|
raise ConnectionError(e, request=request)
|
||
|
|
|
||
|
|
except _ProxyError as e:
|
||
|
|
raise ProxyError(e)
|
||
|
|
|
||
|
|
except (_SSLError, _HTTPError) as e:
|
||
|
|
if isinstance(e, _SSLError):
|
||
|
|
# This branch is for urllib3 versions earlier than v1.22
|
||
|
|
raise SSLError(e, request=request)
|
||
|
|
elif isinstance(e, ReadTimeoutError):
|
||
|
|
raise ReadTimeout(e, request=request)
|
||
|
|
elif isinstance(e, _InvalidHeader):
|
||
|
|
raise InvalidHeader(e, request=request)
|
||
|
|
else:
|
||
|
|
raise
|
||
|
|
|
||
|
|
return self.build_response(request, resp)
|