Merge pull request #38 from cdent/add-urllib3
Add support for intercepting urllib3
This commit is contained in:
commit
b746551b5a
|
@ -15,6 +15,7 @@ Examples
|
|||
http_client
|
||||
httplib2
|
||||
requests
|
||||
urllib3
|
||||
urllib
|
||||
|
||||
|
||||
|
|
|
@ -0,0 +1,32 @@
|
|||
urllib3_intercept
|
||||
==================
|
||||
|
||||
.. automodule:: wsgi_intercept.urllib3_intercept
|
||||
|
||||
|
||||
Example:
|
||||
|
||||
.. testcode::
|
||||
|
||||
import urllib3
|
||||
from wsgi_intercept import urllib3_intercept, add_wsgi_intercept
|
||||
|
||||
pool = urllib3.PoolManager()
|
||||
|
||||
|
||||
def app(environ, start_response):
|
||||
start_response('200 OK', [('Content-Type', 'text/plain')])
|
||||
return [b'Whee']
|
||||
|
||||
|
||||
def make_app():
|
||||
return app
|
||||
|
||||
|
||||
host, port = 'localhost', 80
|
||||
url = 'http://{0}:{1}/'.format(host, port)
|
||||
urllib3_intercept.install()
|
||||
add_wsgi_intercept(host, port, make_app)
|
||||
resp = pool.requests('GET', url)
|
||||
assert resp.data == b'Whee'
|
||||
urllib3_intercept.uninstall()
|
1
setup.py
1
setup.py
|
@ -38,6 +38,7 @@ META = {
|
|||
'pytest>=2.4',
|
||||
'httplib2',
|
||||
'requests>=2.0.1',
|
||||
'urllib3',
|
||||
],
|
||||
},
|
||||
}
|
||||
|
|
|
@ -9,6 +9,7 @@ from uuid import uuid4
|
|||
|
||||
import py.test
|
||||
import requests
|
||||
import urllib3
|
||||
from httplib2 import Http, ServerNotFoundError
|
||||
from six.moves import http_client
|
||||
from six.moves.urllib.request import urlopen
|
||||
|
@ -16,9 +17,10 @@ from six.moves.urllib.error import URLError
|
|||
|
||||
from wsgi_intercept.interceptor import (
|
||||
Interceptor, HttpClientInterceptor, Httplib2Interceptor,
|
||||
RequestsInterceptor, UrllibInterceptor)
|
||||
RequestsInterceptor, UrllibInterceptor, Urllib3Interceptor)
|
||||
from .wsgi_app import simple_app
|
||||
|
||||
httppool = urllib3.PoolManager()
|
||||
|
||||
def app():
|
||||
return simple_app
|
||||
|
@ -178,6 +180,41 @@ def test_requests_in_out():
|
|||
requests.get(url)
|
||||
|
||||
|
||||
# urllib3
|
||||
|
||||
def test_urllib3_interceptor_host():
|
||||
hostname = str(uuid4())
|
||||
port = 9999
|
||||
with Urllib3Interceptor(app=app, host=hostname, port=port) as url:
|
||||
response = httppool.request('GET', url)
|
||||
assert response.status == 200
|
||||
assert 'WSGI intercept successful!' in str(response.data)
|
||||
|
||||
|
||||
def test_urllib3_interceptor_url():
|
||||
hostname = str(uuid4())
|
||||
port = 9999
|
||||
url = 'http://%s:%s/' % (hostname, port)
|
||||
with Urllib3Interceptor(app=app, url=url) as target_url:
|
||||
response = httppool.request('GET', target_url)
|
||||
assert response.status == 200
|
||||
assert 'WSGI intercept successful!' in str(response.data)
|
||||
|
||||
|
||||
def test_urllib3_in_out():
|
||||
hostname = str(uuid4())
|
||||
port = 9999
|
||||
url = 'http://%s:%s/' % (hostname, port)
|
||||
with Urllib3Interceptor(app=app, url=url) as target_url:
|
||||
response = httppool.request('GET', target_url)
|
||||
assert response.status == 200
|
||||
assert 'WSGI intercept successful!' in str(response.data)
|
||||
|
||||
# outside the context manager the intercept does not work
|
||||
with py.test.raises(urllib3.exceptions.MaxRetryError):
|
||||
httppool.request('GET', url)
|
||||
|
||||
|
||||
# urllib
|
||||
|
||||
def test_urllib_interceptor_host():
|
||||
|
|
|
@ -0,0 +1,90 @@
|
|||
import os
|
||||
import py.test
|
||||
import socket
|
||||
from wsgi_intercept import urllib3_intercept, WSGIAppError
|
||||
from test import wsgi_app
|
||||
from test.install import installer_class, skipnetwork
|
||||
import urllib3
|
||||
|
||||
HOST = 'some_hopefully_nonexistant_domain'
|
||||
|
||||
InstalledApp = installer_class(urllib3_intercept)
|
||||
http = urllib3.PoolManager()
|
||||
|
||||
|
||||
def test_http():
|
||||
with InstalledApp(wsgi_app.simple_app, host=HOST, port=80) as app:
|
||||
resp = http.request('GET', 'http://some_hopefully_nonexistant_domain:80/')
|
||||
assert resp.data == b'WSGI intercept successful!\n'
|
||||
assert app.success()
|
||||
|
||||
|
||||
def test_http_default_port():
|
||||
with InstalledApp(wsgi_app.simple_app, host=HOST, port=80) as app:
|
||||
resp = http.request('GET', 'http://some_hopefully_nonexistant_domain/')
|
||||
assert resp.data == b'WSGI intercept successful!\n'
|
||||
assert app.success()
|
||||
|
||||
|
||||
def test_http_other_port():
|
||||
with InstalledApp(wsgi_app.simple_app, host=HOST, port=8080) as app:
|
||||
resp = http.request('GET', 'http://some_hopefully_nonexistant_domain:8080/')
|
||||
assert resp.data == b'WSGI intercept successful!\n'
|
||||
assert app.success()
|
||||
environ = app.get_internals()
|
||||
assert environ['wsgi.url_scheme'] == 'http'
|
||||
|
||||
|
||||
def test_bogus_domain():
|
||||
with InstalledApp(wsgi_app.simple_app, host=HOST, port=80):
|
||||
py.test.raises(
|
||||
urllib3.exceptions.MaxRetryError,
|
||||
'http.request("GET", "http://_nonexistant_domain_")')
|
||||
|
||||
|
||||
def test_proxy_handling():
|
||||
with py.test.raises(RuntimeError) as exc:
|
||||
with InstalledApp(wsgi_app.simple_app, host=HOST, port=80,
|
||||
proxy='some_proxy.com:1234'):
|
||||
http.request('GET', 'http://some_hopefully_nonexistant_domain:80/')
|
||||
assert 'http_proxy or https_proxy set in environment' in str(exc.value)
|
||||
# We need to do this by hand because the exception was raised
|
||||
# during the entry of the context manager, so the exit handler
|
||||
# wasn't reached.
|
||||
del os.environ['http_proxy']
|
||||
|
||||
|
||||
def test_https():
|
||||
with InstalledApp(wsgi_app.simple_app, host=HOST, port=443) as app:
|
||||
resp = http.request('GET', 'https://some_hopefully_nonexistant_domain:443/')
|
||||
assert resp.data == b'WSGI intercept successful!\n'
|
||||
assert app.success()
|
||||
|
||||
|
||||
def test_https_default_port():
|
||||
with InstalledApp(wsgi_app.simple_app, host=HOST, port=443) as app:
|
||||
resp = http.request('GET', 'https://some_hopefully_nonexistant_domain/')
|
||||
assert resp.data == b'WSGI intercept successful!\n'
|
||||
assert app.success()
|
||||
environ = app.get_internals()
|
||||
assert environ['wsgi.url_scheme'] == 'https'
|
||||
|
||||
|
||||
def test_app_error():
|
||||
with InstalledApp(wsgi_app.raises_app, host=HOST, port=80):
|
||||
with py.test.raises(WSGIAppError):
|
||||
http.request('GET', 'http://some_hopefully_nonexistant_domain/')
|
||||
|
||||
|
||||
@skipnetwork
|
||||
def test_http_not_intercepted():
|
||||
with InstalledApp(wsgi_app.raises_app, host=HOST, port=80):
|
||||
resp = http.request('GET', 'http://google.com')
|
||||
assert resp.status >= 200 and resp.status < 300
|
||||
|
||||
|
||||
@skipnetwork
|
||||
def test_https_not_intercepted():
|
||||
with InstalledApp(wsgi_app.raises_app, host=HOST, port=80):
|
||||
resp = http.request('GET', 'https://google.com')
|
||||
assert resp.status >= 200 and resp.status < 300
|
|
@ -0,0 +1,45 @@
|
|||
"""Common code of urllib3 and requests intercepts."""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
from . import WSGI_HTTPConnection, WSGI_HTTPSConnection, wsgi_fake_socket
|
||||
|
||||
|
||||
wsgi_fake_socket.settimeout = lambda self, timeout: None
|
||||
|
||||
|
||||
def make_urllib3_override(HTTPConnectionPool, HTTPSConnectionPool,
|
||||
HTTPConnection, HTTPSConnection):
|
||||
|
||||
class HTTP_WSGIInterceptor(WSGI_HTTPConnection, HTTPConnection):
|
||||
def __init__(self, *args, **kwargs):
|
||||
if 'strict' in kwargs and sys.version_info > (3, 0):
|
||||
kwargs.pop('strict')
|
||||
WSGI_HTTPConnection.__init__(self, *args, **kwargs)
|
||||
HTTPConnection.__init__(self, *args, **kwargs)
|
||||
|
||||
|
||||
class HTTPS_WSGIInterceptor(WSGI_HTTPSConnection, HTTPSConnection):
|
||||
is_verified = True
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
if 'strict' in kwargs and sys.version_info > (3, 0):
|
||||
kwargs.pop('strict')
|
||||
WSGI_HTTPSConnection.__init__(self, *args, **kwargs)
|
||||
HTTPSConnection.__init__(self, *args, **kwargs)
|
||||
|
||||
|
||||
def install():
|
||||
if 'http_proxy' in os.environ or 'https_proxy' in os.environ:
|
||||
raise RuntimeError(
|
||||
'http_proxy or https_proxy set in environment, please unset')
|
||||
HTTPConnectionPool.ConnectionCls = HTTP_WSGIInterceptor
|
||||
HTTPSConnectionPool.ConnectionCls = HTTPS_WSGIInterceptor
|
||||
|
||||
|
||||
def uninstall():
|
||||
HTTPConnectionPool.ConnectionCls = HTTPConnection
|
||||
HTTPSConnectionPool.ConnectionCls = HTTPSConnection
|
||||
|
||||
return install, uninstall
|
|
@ -109,6 +109,12 @@ class RequestsInterceptor(Interceptor):
|
|||
MODULE_NAME = 'requests_intercept'
|
||||
|
||||
|
||||
class Urllib3Interceptor(Interceptor):
|
||||
"""Interceptor for requests."""
|
||||
|
||||
MODULE_NAME = 'urllib3_intercept'
|
||||
|
||||
|
||||
class UrllibInterceptor(Interceptor):
|
||||
"""Interceptor for urllib2 and urllib.request."""
|
||||
|
||||
|
|
|
@ -2,45 +2,14 @@
|
|||
`requests <http://docs.python-requests.org/en/latest/>`_.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
from . import WSGI_HTTPConnection, WSGI_HTTPSConnection, wsgi_fake_socket
|
||||
from requests.packages.urllib3.connectionpool import (HTTPConnectionPool,
|
||||
HTTPSConnectionPool)
|
||||
from requests.packages.urllib3.connection import (HTTPConnection,
|
||||
HTTPSConnection)
|
||||
from ._urllib3 import make_urllib3_override
|
||||
|
||||
|
||||
wsgi_fake_socket.settimeout = lambda self, timeout: None
|
||||
|
||||
|
||||
class HTTP_WSGIInterceptor(WSGI_HTTPConnection, HTTPConnection):
|
||||
def __init__(self, *args, **kwargs):
|
||||
if 'strict' in kwargs and sys.version_info > (3, 0):
|
||||
kwargs.pop('strict')
|
||||
WSGI_HTTPConnection.__init__(self, *args, **kwargs)
|
||||
HTTPConnection.__init__(self, *args, **kwargs)
|
||||
|
||||
|
||||
class HTTPS_WSGIInterceptor(WSGI_HTTPSConnection, HTTPSConnection):
|
||||
is_verified = True
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
if 'strict' in kwargs and sys.version_info > (3, 0):
|
||||
kwargs.pop('strict')
|
||||
WSGI_HTTPSConnection.__init__(self, *args, **kwargs)
|
||||
HTTPSConnection.__init__(self, *args, **kwargs)
|
||||
|
||||
|
||||
def install():
|
||||
if 'http_proxy' in os.environ or 'https_proxy' in os.environ:
|
||||
raise RuntimeError(
|
||||
'http_proxy or https_proxy set in environment, please unset')
|
||||
HTTPConnectionPool.ConnectionCls = HTTP_WSGIInterceptor
|
||||
HTTPSConnectionPool.ConnectionCls = HTTPS_WSGIInterceptor
|
||||
|
||||
|
||||
def uninstall():
|
||||
HTTPConnectionPool.ConnectionCls = HTTPConnection
|
||||
HTTPSConnectionPool.ConnectionCls = HTTPSConnection
|
||||
install, uninstall = make_urllib3_override(HTTPConnectionPool,
|
||||
HTTPSConnectionPool,
|
||||
HTTPConnection,
|
||||
HTTPSConnection)
|
||||
|
|
|
@ -0,0 +1,13 @@
|
|||
"""Intercept HTTP connections that use
|
||||
`urllib3 <https://urllib3.readthedocs.org/>`_.
|
||||
"""
|
||||
|
||||
from urllib3.connectionpool import HTTPConnectionPool, HTTPSConnectionPool
|
||||
from urllib3.connection import HTTPConnection, HTTPSConnection
|
||||
from ._urllib3 import make_urllib3_override
|
||||
|
||||
|
||||
install, uninstall = make_urllib3_override(HTTPConnectionPool,
|
||||
HTTPSConnectionPool,
|
||||
HTTPConnection,
|
||||
HTTPSConnection)
|
Loading…
Reference in New Issue