from __future__
import absolute_import
import collections
import functools
import logging
from ._collections
import RecentlyUsedContainer
from .connectionpool
import HTTPConnectionPool, HTTPSConnectionPool, port_by_scheme
from .exceptions
import (
LocationValueError,
MaxRetryError,
ProxySchemeUnknown,
ProxySchemeUnsupported,
URLSchemeUnknown,
)
from .packages
import six
from .packages.six.moves.urllib.parse
import urljoin
from .request
import RequestMethods
from .util.proxy
import connection_requires_http_tunnel
from .util.retry
import Retry
from .util.url
import parse_url
__all__ = [
"PoolManager",
"ProxyManager",
"proxy_from_url"]
log = logging.getLogger(__name__)
SSL_KEYWORDS = (
"key_file",
"cert_file",
"cert_reqs",
"ca_certs",
"ssl_version",
"ca_cert_dir",
"ssl_context",
"key_password",
"server_hostname",
)
# All known keyword arguments that could be provided to the pool manager, its
# pools, or the underlying connections. This is used to construct a pool key.
_key_fields = (
"key_scheme",
# str
"key_host",
# str
"key_port",
# int
"key_timeout",
# int or float or Timeout
"key_retries",
# int or Retry
"key_strict",
# bool
"key_block",
# bool
"key_source_address",
# str
"key_key_file",
# str
"key_key_password",
# str
"key_cert_file",
# str
"key_cert_reqs",
# str
"key_ca_certs",
# str
"key_ssl_version",
# str
"key_ca_cert_dir",
# str
"key_ssl_context",
# instance of ssl.SSLContext or urllib3.util.ssl_.SSLContext
"key_maxsize",
# int
"key_headers",
# dict
"key__proxy",
# parsed proxy url
"key__proxy_headers",
# dict
"key__proxy_config",
# class
"key_socket_options",
# list of (level (int), optname (int), value (int or str)) tuples
"key__socks_options",
# dict
"key_assert_hostname",
# bool or string
"key_assert_fingerprint",
# str
"key_server_hostname",
# str
)
#: The namedtuple class used to construct keys for the connection pool.
#: All custom key schemes should include the fields in this key at a minimum.
PoolKey = collections.namedtuple(
"PoolKey", _key_fields)
_proxy_config_fields = (
"ssl_context",
"use_forwarding_for_https")
ProxyConfig = collections.namedtuple(
"ProxyConfig", _proxy_config_fields)
def _default_key_normalizer(key_class, request_context):
"""
Create a pool key out of a request context dictionary.
According to RFC 3986, both the scheme
and host are case-insensitive.
Therefore, this function normalizes both before constructing the pool
key
for an HTTPS request.
If you wish to change this behaviour, provide
alternate callables to ``key_fn_by_scheme``.
:param key_class:
The
class to use when constructing the key. This should be a namedtuple
with the ``scheme``
and ``host`` keys at a minimum.
:type key_class: namedtuple
:param request_context:
A dictionary-like object that contain the context
for a request.
:type request_context: dict
:
return: A namedtuple that can be used
as a connection pool key.
:rtype: PoolKey
"""
# Since we mutate the dictionary, make a copy first
context = request_context.copy()
context[
"scheme"] = context[
"scheme"].lower()
context[
"host"] = context[
"host"].lower()
# These are both dictionaries and need to be transformed into frozensets
for key
in (
"headers",
"_proxy_headers",
"_socks_options"):
if key
in context
and context[key]
is not None:
context[key] = frozenset(context[key].items())
# The socket_options key may be a list and needs to be transformed into a
# tuple.
socket_opts = context.get(
"socket_options")
if socket_opts
is not None:
context[
"socket_options"] = tuple(socket_opts)
# Map the kwargs to the names in the namedtuple - this is necessary since
# namedtuples can't have fields starting with '_'.
for key
in list(context.keys()):
context[
"key_" + key] = context.pop(key)
# Default to ``None`` for keys missing from the context
for field
in key_class._fields:
if field
not in context:
context[field] =
None
return key_class(**context)
#: A dictionary that maps a scheme to a callable that creates a pool key.
#: This can be used to alter the way pool keys are constructed, if desired.
#: Each PoolManager makes a copy of this dictionary so they can be configured
#: globally here, or individually on the instance.
key_fn_by_scheme = {
"http": functools.partial(_default_key_normalizer, PoolKey),
"https": functools.partial(_default_key_normalizer, PoolKey),
}
pool_classes_by_scheme = {
"http": HTTPConnectionPool,
"https": HTTPSConnectionPool}
class PoolManager(RequestMethods):
"""
Allows
for arbitrary requests
while transparently keeping track of
necessary connection pools
for you.
:param num_pools:
Number of connection pools to cache before discarding the least
recently used pool.
:param headers:
Headers to include
with all requests, unless other headers are given
explicitly.
:param \\**connection_pool_kw:
Additional parameters are used to create fresh
:
class:`urllib3.connectionpool.ConnectionPool` instances.
Example::
>>> manager = PoolManager(num_pools=2)
>>> r = manager.request(
'GET',
'http://google.com/')
>>> r = manager.request(
'GET',
'http://google.com/mail')
>>> r = manager.request(
'GET',
'http://yahoo.com/')
>>> len(manager.pools)
2
"""
proxy =
None
proxy_config =
None
def __init__(self, num_pools=10, headers=
None, **connection_pool_kw):
RequestMethods.__init__(self, headers)
self.connection_pool_kw = connection_pool_kw
self.pools = RecentlyUsedContainer(num_pools)
# Locally set the pool classes and keys so other PoolManagers can
# override them.
self.pool_classes_by_scheme = pool_classes_by_scheme
self.key_fn_by_scheme = key_fn_by_scheme.copy()
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.clear()
# Return False to re-raise any potential exceptions
return False
def _new_pool(self, scheme, host, port, request_context=
None):
"""
Create a new :
class:`urllib3.connectionpool.ConnectionPool` based on host, port, scheme,
and
any additional pool keyword arguments.
If ``request_context``
is provided, it
is provided
as keyword arguments
to the pool
class used. This method
is used to actually create the
connection pools handed out by :meth:`connection_from_url`
and
companion methods. It
is intended to be overridden
for customization.
"""
pool_cls = self.pool_classes_by_scheme[scheme]
if request_context
is None:
request_context = self.connection_pool_kw.copy()
# Although the context has everything necessary to create the pool,
# this function has historically only used the scheme, host, and port
# in the positional args. When an API change is acceptable these can
# be removed.
for key
in (
"scheme",
"host",
"port"):
request_context.pop(key,
None)
if scheme ==
"http":
for kw
in SSL_KEYWORDS:
request_context.pop(kw,
None)
return pool_cls(host, port, **request_context)
def clear(self):
"""
Empty our store of pools
and direct them all to close.
This will
not affect in-flight connections, but they will
not be
re-used after completion.
"""
self.pools.clear()
def connection_from_host(self, host, port=
None, scheme=
"http", pool_kwargs=
None):
"""
Get a :
class:`urllib3.connectionpool.ConnectionPool` based on the host, port,
and scheme.
If ``port`` isn
't given, it will be derived from the ``scheme`` using
``urllib3.connectionpool.port_by_scheme``.
If ``pool_kwargs``
is
provided, it
is merged
with the instance
's ``connection_pool_kw``
variable
and used to create the new connection pool,
if one
is
needed.
"""
if not host:
raise LocationValueError(
"No host specified.")
request_context = self._merge_pool_kwargs(pool_kwargs)
request_context[
"scheme"] = scheme
or "http"
if not port:
port = port_by_scheme.get(request_context[
"scheme"].lower(), 80)
request_context[
"port"] = port
request_context[
"host"] = host
return self.connection_from_context(request_context)
def connection_from_context(self, request_context):
"""
Get a :
class:`urllib3.connectionpool.ConnectionPool` based on the request context.
``request_context`` must at least contain the ``scheme`` key
and its
value must be a key
in ``key_fn_by_scheme`` instance variable.
"""
scheme = request_context[
"scheme"].lower()
pool_key_constructor = self.key_fn_by_scheme.get(scheme)
if not pool_key_constructor:
raise URLSchemeUnknown(scheme)
pool_key = pool_key_constructor(request_context)
return self.connection_from_pool_key(pool_key, request_context=request_context)
def connection_from_pool_key(self, pool_key, request_context=
None):
"""
Get a :
class:`urllib3.connectionpool.ConnectionPool` based on the provided pool key.
``pool_key`` should be a namedtuple that only contains immutable
objects. At a minimum it must have the ``scheme``, ``host``,
and
``port`` fields.
"""
with self.pools.lock:
# If the scheme, host, or port doesn't match existing open
# connections, open a new ConnectionPool.
pool = self.pools.get(pool_key)
if pool:
return pool
# Make a fresh ConnectionPool of the desired type
scheme = request_context[
"scheme"]
host = request_context[
"host"]
port = request_context[
"port"]
pool = self._new_pool(scheme, host, port, request_context=request_context)
self.pools[pool_key] = pool
return pool
def connection_from_url(self, url, pool_kwargs=
None):
"""
Similar to :func:`urllib3.connectionpool.connection_from_url`.
If ``pool_kwargs``
is not provided
and a new pool needs to be
constructed, ``self.connection_pool_kw``
is used to initialize
the :
class:`urllib3.connectionpool.ConnectionPool`.
If ``pool_kwargs``
is provided, it
is used instead. Note that
if a new pool does
not
need to be created
for the request, the provided ``pool_kwargs`` are
not used.
"""
u = parse_url(url)
return self.connection_from_host(
u.host, port=u.port, scheme=u.scheme, pool_kwargs=pool_kwargs
)
def _merge_pool_kwargs(self, override):
"""
Merge a dictionary of override values
for self.connection_pool_kw.
This does
not modify self.connection_pool_kw
and returns a new dict.
Any keys
in the override dictionary
with a value of ``
None`` are
removed
from the merged dictionary.
"""
base_pool_kwargs = self.connection_pool_kw.copy()
if override:
for key, value
in override.items():
if value
is None:
try:
del base_pool_kwargs[key]
except KeyError:
pass
else:
base_pool_kwargs[key] = value
return base_pool_kwargs
def _proxy_requires_url_absolute_form(self, parsed_url):
"""
Indicates
if the proxy requires the complete destination URL
in the
request. Normally this
is only needed when
not using an HTTP CONNECT
tunnel.
"""
if self.proxy
is None:
return False
return not connection_requires_http_tunnel(
self.proxy, self.proxy_config, parsed_url.scheme
)
def _validate_proxy_scheme_url_selection(self, url_scheme):
"""
Validates that were
not attempting to do TLS
in TLS connections on
Python2
or with unsupported SSL implementations.
"""
if self.proxy
is None or url_scheme !=
"https":
return
if self.proxy.scheme !=
"https":
return
if six.PY2
and not self.proxy_config.use_forwarding_for_https:
raise ProxySchemeUnsupported(
"Contacting HTTPS destinations through HTTPS proxies "
"'via CONNECT tunnels' is not supported in Python 2"
)
def urlopen(self, method, url, redirect=
True, **kw):
"""
Same
as :meth:`urllib3.HTTPConnectionPool.urlopen`
with custom cross-host redirect logic
and only sends the request-uri
portion of the ``url``.
The given ``url`` parameter must be absolute, such that an appropriate
:
class:`urllib3.connectionpool.ConnectionPool` can be chosen
for it.
"""
u = parse_url(url)
self._validate_proxy_scheme_url_selection(u.scheme)
conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme)
kw[
"assert_same_host"] =
False
kw[
"redirect"] =
False
if "headers" not in kw:
kw[
"headers"] = self.headers.copy()
if self._proxy_requires_url_absolute_form(u):
response = conn.urlopen(method, url, **kw)
else:
response = conn.urlopen(method, u.request_uri, **kw)
redirect_location = redirect
and response.get_redirect_location()
if not redirect_location:
return response
# Support relative URLs for redirecting.
redirect_location = urljoin(url, redirect_location)
# RFC 7231, Section 6.4.4
if response.status == 303:
method =
"GET"
retries = kw.get(
"retries")
if not isinstance(retries, Retry):
retries = Retry.from_int(retries, redirect=redirect)
# Strip headers marked as unsafe to forward to the redirected location.
# Check remove_headers_on_redirect to avoid a potential network call within
# conn.is_same_host() which may use socket.gethostbyname() in the future.
if retries.remove_headers_on_redirect
and not conn.is_same_host(
redirect_location
):
headers = list(six.iterkeys(kw[
"headers"]))
for header
in headers:
if header.lower()
in retries.remove_headers_on_redirect:
kw[
"headers"].pop(header,
None)
try:
retries = retries.increment(method, url, response=response, _pool=conn)
except MaxRetryError:
if retries.raise_on_redirect:
response.drain_conn()
raise
return response
kw[
"retries"] = retries
kw[
"redirect"] = redirect
log.info(
"Redirecting %s -> %s", url, redirect_location)
response.drain_conn()
return self.urlopen(method, redirect_location, **kw)
class ProxyManager(PoolManager):
"""
Behaves just like :
class:`PoolManager`, but sends all requests through
the defined proxy, using the CONNECT method
for HTTPS URLs.
:param proxy_url:
The URL of the proxy to be used.
:param proxy_headers:
A dictionary containing headers that will be sent to the proxy.
In case
of HTTP they are being sent
with each request,
while in the
HTTPS/CONNECT case they are sent only once. Could be used
for proxy
authentication.
:param proxy_ssl_context:
The proxy SSL context
is used to establish the TLS connection to the
proxy when using HTTPS proxies.
:param use_forwarding_for_https:
(Defaults to
False)
If set to
True will forward requests to the HTTPS
proxy to be made on behalf of the client instead of creating a TLS
tunnel via the CONNECT method. **Enabling this flag means that request
and response headers
and content will be visible
from the HTTPS proxy**
whereas tunneling keeps request
and response headers
and content
private. IP address, target hostname, SNI,
and port are always visible
to an HTTPS proxy even when this flag
is disabled.
Example:
>>> proxy = urllib3.ProxyManager(
'http://localhost:3128/')
>>> r1 = proxy.request(
'GET',
'http://google.com/')
>>> r2 = proxy.request(
'GET',
'http://httpbin.org/')
>>> len(proxy.pools)
1
>>> r3 = proxy.request(
'GET',
'https://httpbin.org/')
>>> r4 = proxy.request(
'GET',
'https://twitter.com/')
>>> len(proxy.pools)
3
"""
def __init__(
self,
proxy_url,
num_pools=10,
headers=
None,
proxy_headers=
None,
proxy_ssl_context=
None,
use_forwarding_for_https=
False,
**connection_pool_kw
):
if isinstance(proxy_url, HTTPConnectionPool):
proxy_url =
"%s://%s:%i" % (
proxy_url.scheme,
proxy_url.host,
proxy_url.port,
)
proxy = parse_url(proxy_url)
if proxy.scheme
not in (
"http",
"https"):
raise ProxySchemeUnknown(proxy.scheme)
if not proxy.port:
port = port_by_scheme.get(proxy.scheme, 80)
proxy = proxy._replace(port=port)
self.proxy = proxy
self.proxy_headers = proxy_headers
or {}
self.proxy_ssl_context = proxy_ssl_context
self.proxy_config = ProxyConfig(proxy_ssl_context, use_forwarding_for_https)
connection_pool_kw[
"_proxy"] = self.proxy
connection_pool_kw[
"_proxy_headers"] = self.proxy_headers
connection_pool_kw[
"_proxy_config"] = self.proxy_config
super(ProxyManager, self).__init__(num_pools, headers, **connection_pool_kw)
def connection_from_host(self, host, port=
None, scheme=
"http", pool_kwargs=
None):
if scheme ==
"https":
return super(ProxyManager, self).connection_from_host(
host, port, scheme, pool_kwargs=pool_kwargs
)
return super(ProxyManager, self).connection_from_host(
self.proxy.host, self.proxy.port, self.proxy.scheme, pool_kwargs=pool_kwargs
)
def _set_proxy_headers(self, url, headers=
None):
"""
Sets headers needed by proxies: specifically, the Accept
and Host
headers. Only sets headers
not provided by the user.
"""
headers_ = {
"Accept":
"*/*"}
netloc = parse_url(url).netloc
if netloc:
headers_[
"Host"] = netloc
if headers:
headers_.update(headers)
return headers_
def urlopen(self, method, url, redirect=
True, **kw):
"Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute."
u = parse_url(url)
if not connection_requires_http_tunnel(self.proxy, self.proxy_config, u.scheme):
# For connections using HTTP CONNECT, httplib sets the necessary
# headers on the CONNECT to the proxy. If we're not using CONNECT,
# we'll definitely need to set 'Host' at the very least.
headers = kw.get(
"headers", self.headers)
kw[
"headers"] = self._set_proxy_headers(url, headers)
return super(ProxyManager, self).urlopen(method, url, redirect=redirect, **kw)
def proxy_from_url(url, **kw):
return ProxyManager(proxy_url=url, **kw)
