From fbfbfa36903f7086f59ce63679826eb057f4d15f Mon Sep 17 00:00:00 2001 From: Chris Dent Date: Fri, 18 Mar 2016 13:04:44 +0000 Subject: [PATCH 1/5] Add support for intercepting urllib3 This is relatively straightforward but presents a problem. The monkeypatching is identical as that for requests, but on different modules, because of the vendorization of urllib3 that requests does. It would be better if there was some way to have just one set of code that removes the duplication and does the right but maintains the semantics of intercepting the desired thing by name. --- test/test_interceptor.py | 39 ++++++++++++- test/test_urllib3.py | 90 +++++++++++++++++++++++++++++ wsgi_intercept/interceptor.py | 6 ++ wsgi_intercept/urllib3_intercept.py | 46 +++++++++++++++ 4 files changed, 180 insertions(+), 1 deletion(-) create mode 100644 test/test_urllib3.py create mode 100644 wsgi_intercept/urllib3_intercept.py diff --git a/test/test_interceptor.py b/test/test_interceptor.py index ca37ebd..9e986d5 100644 --- a/test/test_interceptor.py +++ b/test/test_interceptor.py @@ -9,6 +9,7 @@ from uuid import uuid4 import py.test import requests +import urllib3 from httplib2 import Http, ServerNotFoundError from six.moves import http_client from six.moves.urllib.request import urlopen @@ -16,9 +17,10 @@ from six.moves.urllib.error import URLError from wsgi_intercept.interceptor import ( Interceptor, HttpClientInterceptor, Httplib2Interceptor, - RequestsInterceptor, UrllibInterceptor) + RequestsInterceptor, UrllibInterceptor, Urllib3Interceptor) from .wsgi_app import simple_app +httppool = urllib3.PoolManager() def app(): return simple_app @@ -178,6 +180,41 @@ def test_requests_in_out(): requests.get(url) +# urllib3 + +def test_urllib3_interceptor_host(): + hostname = str(uuid4()) + port = 9999 + with Urllib3Interceptor(app=app, host=hostname, port=port) as url: + response = httppool.request('GET', url) + assert response.status == 200 + assert 'WSGI intercept successful!' in response.data + + +def test_urllib3_interceptor_url(): + hostname = str(uuid4()) + port = 9999 + url = 'http://%s:%s/' % (hostname, port) + with Urllib3Interceptor(app=app, url=url) as target_url: + response = httppool.request('GET', target_url) + assert response.status == 200 + assert 'WSGI intercept successful!' in response.data + + +def test_urllib3_in_out(): + hostname = str(uuid4()) + port = 9999 + url = 'http://%s:%s/' % (hostname, port) + with Urllib3Interceptor(app=app, url=url) as target_url: + response = httppool.request('GET', target_url) + assert response.status == 200 + assert 'WSGI intercept successful!' in response.data + + # outside the context manager the intercept does not work + with py.test.raises(urllib3.exceptions.MaxRetryError): + httppool.request('GET', url) + + # urllib def test_urllib_interceptor_host(): diff --git a/test/test_urllib3.py b/test/test_urllib3.py new file mode 100644 index 0000000..8206d5e --- /dev/null +++ b/test/test_urllib3.py @@ -0,0 +1,90 @@ +import os +import py.test +import socket +from wsgi_intercept import urllib3_intercept, WSGIAppError +from test import wsgi_app +from test.install import installer_class, skipnetwork +import urllib3 + +HOST = 'some_hopefully_nonexistant_domain' + +InstalledApp = installer_class(urllib3_intercept) +http = urllib3.PoolManager() + + +def test_http(): + with InstalledApp(wsgi_app.simple_app, host=HOST, port=80) as app: + resp = http.request('GET', 'http://some_hopefully_nonexistant_domain:80/') + assert resp.data == b'WSGI intercept successful!\n' + assert app.success() + + +def test_http_default_port(): + with InstalledApp(wsgi_app.simple_app, host=HOST, port=80) as app: + resp = http.request('GET', 'http://some_hopefully_nonexistant_domain/') + assert resp.data == b'WSGI intercept successful!\n' + assert app.success() + + +def test_http_other_port(): + with InstalledApp(wsgi_app.simple_app, host=HOST, port=8080) as app: + resp = http.request('GET', 'http://some_hopefully_nonexistant_domain:8080/') + assert resp.data == b'WSGI intercept successful!\n' + assert app.success() + environ = app.get_internals() + assert environ['wsgi.url_scheme'] == 'http' + + +def test_bogus_domain(): + with InstalledApp(wsgi_app.simple_app, host=HOST, port=80): + py.test.raises( + urllib3.exceptions.MaxRetryError, + 'http.request("GET", "http://_nonexistant_domain_")') + + +def test_proxy_handling(): + with py.test.raises(RuntimeError) as exc: + with InstalledApp(wsgi_app.simple_app, host=HOST, port=80, + proxy='some_proxy.com:1234'): + http.request('GET', 'http://some_hopefully_nonexistant_domain:80/') + assert 'http_proxy or https_proxy set in environment' in str(exc.value) + # We need to do this by hand because the exception was raised + # during the entry of the context manager, so the exit handler + # wasn't reached. + del os.environ['http_proxy'] + + +def test_https(): + with InstalledApp(wsgi_app.simple_app, host=HOST, port=443) as app: + resp = http.request('GET', 'https://some_hopefully_nonexistant_domain:443/') + assert resp.data == b'WSGI intercept successful!\n' + assert app.success() + + +def test_https_default_port(): + with InstalledApp(wsgi_app.simple_app, host=HOST, port=443) as app: + resp = http.request('GET', 'https://some_hopefully_nonexistant_domain/') + assert resp.data == b'WSGI intercept successful!\n' + assert app.success() + environ = app.get_internals() + assert environ['wsgi.url_scheme'] == 'https' + + +def test_app_error(): + with InstalledApp(wsgi_app.raises_app, host=HOST, port=80): + with py.test.raises(WSGIAppError): + http.request('GET', 'http://some_hopefully_nonexistant_domain/') + + +@skipnetwork +def test_http_not_intercepted(): + with InstalledApp(wsgi_app.raises_app, host=HOST, port=80): + resp = http.request('GET', 'http://google.com') + assert resp.status >= 200 and resp.status < 300 + + +@skipnetwork +def test_https_not_intercepted(): + with InstalledApp(wsgi_app.raises_app, host=HOST, port=80): + resp = http.request('GET', 'https://google.com') + assert resp.status >= 200 and resp.status < 300 diff --git a/wsgi_intercept/interceptor.py b/wsgi_intercept/interceptor.py index fd48c1a..3daea37 100644 --- a/wsgi_intercept/interceptor.py +++ b/wsgi_intercept/interceptor.py @@ -109,6 +109,12 @@ class RequestsInterceptor(Interceptor): MODULE_NAME = 'requests_intercept' +class Urllib3Interceptor(Interceptor): + """Interceptor for requests.""" + + MODULE_NAME = 'urllib3_intercept' + + class UrllibInterceptor(Interceptor): """Interceptor for urllib2 and urllib.request.""" diff --git a/wsgi_intercept/urllib3_intercept.py b/wsgi_intercept/urllib3_intercept.py new file mode 100644 index 0000000..bb7b377 --- /dev/null +++ b/wsgi_intercept/urllib3_intercept.py @@ -0,0 +1,46 @@ +"""Intercept HTTP connections that use +`urllib3 `_. +""" + +import os +import sys + +# TODO: This is a a total dupe of requests. Need to remove +# duplication. +from . import WSGI_HTTPConnection, WSGI_HTTPSConnection, wsgi_fake_socket +from urllib3.connectionpool import HTTPConnectionPool, HTTPSConnectionPool +from urllib3.connection import HTTPConnection, HTTPSConnection + + +wsgi_fake_socket.settimeout = lambda self, timeout: None + + +class HTTP_WSGIInterceptor(WSGI_HTTPConnection, HTTPConnection): + def __init__(self, *args, **kwargs): + if 'strict' in kwargs and sys.version_info > (3, 0): + kwargs.pop('strict') + WSGI_HTTPConnection.__init__(self, *args, **kwargs) + HTTPConnection.__init__(self, *args, **kwargs) + + +class HTTPS_WSGIInterceptor(WSGI_HTTPSConnection, HTTPSConnection): + is_verified = True + + def __init__(self, *args, **kwargs): + if 'strict' in kwargs and sys.version_info > (3, 0): + kwargs.pop('strict') + WSGI_HTTPSConnection.__init__(self, *args, **kwargs) + HTTPSConnection.__init__(self, *args, **kwargs) + + +def install(): + if 'http_proxy' in os.environ or 'https_proxy' in os.environ: + raise RuntimeError( + 'http_proxy or https_proxy set in environment, please unset') + HTTPConnectionPool.ConnectionCls = HTTP_WSGIInterceptor + HTTPSConnectionPool.ConnectionCls = HTTPS_WSGIInterceptor + + +def uninstall(): + HTTPConnectionPool.ConnectionCls = HTTPConnection + HTTPSConnectionPool.ConnectionCls = HTTPSConnection From 13cd3a436177f6071aceb9aca4bd620ee67e9139 Mon Sep 17 00:00:00 2001 From: Chris Dent Date: Fri, 18 Mar 2016 13:11:24 +0000 Subject: [PATCH 2/5] add urllib3 to testing requires --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 53037cd..3925630 100644 --- a/setup.py +++ b/setup.py @@ -38,6 +38,7 @@ META = { 'pytest>=2.4', 'httplib2', 'requests>=2.0.1', + 'urllib3', ], }, } From 8e02d4e1fe18a2f2099261cd086cbcd77b7a1726 Mon Sep 17 00:00:00 2001 From: Chris Dent Date: Fri, 18 Mar 2016 13:24:10 +0000 Subject: [PATCH 3/5] Fix for python3 in use of urllib3.Response.data Is bytes, we need to cast it to str to do `in`. --- test/test_interceptor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_interceptor.py b/test/test_interceptor.py index 9e986d5..66e0881 100644 --- a/test/test_interceptor.py +++ b/test/test_interceptor.py @@ -188,7 +188,7 @@ def test_urllib3_interceptor_host(): with Urllib3Interceptor(app=app, host=hostname, port=port) as url: response = httppool.request('GET', url) assert response.status == 200 - assert 'WSGI intercept successful!' in response.data + assert 'WSGI intercept successful!' in str(response.data) def test_urllib3_interceptor_url(): @@ -198,7 +198,7 @@ def test_urllib3_interceptor_url(): with Urllib3Interceptor(app=app, url=url) as target_url: response = httppool.request('GET', target_url) assert response.status == 200 - assert 'WSGI intercept successful!' in response.data + assert 'WSGI intercept successful!' in str(response.data) def test_urllib3_in_out(): @@ -208,7 +208,7 @@ def test_urllib3_in_out(): with Urllib3Interceptor(app=app, url=url) as target_url: response = httppool.request('GET', target_url) assert response.status == 200 - assert 'WSGI intercept successful!' in response.data + assert 'WSGI intercept successful!' in str(response.data) # outside the context manager the intercept does not work with py.test.raises(urllib3.exceptions.MaxRetryError): From c464f2ec7a16fe074daef9ebb9ec1c04cc0dc98c Mon Sep 17 00:00:00 2001 From: Chris Dent Date: Fri, 18 Mar 2016 13:43:34 +0000 Subject: [PATCH 4/5] Dedupe the urllib3 monkey patching code requests vendorizes urllib3 which makes monkey patching it while also wanting to monkey patch proper urllib3 a bit hairy. Here we use a factory to do the overrides for us. Hat tip to @FND for helping with the thinking through of this. --- wsgi_intercept/_urllib3.py | 45 ++++++++++++++++++++++++++++ wsgi_intercept/requests_intercept.py | 41 ++++--------------------- wsgi_intercept/urllib3_intercept.py | 43 ++++---------------------- 3 files changed, 55 insertions(+), 74 deletions(-) create mode 100644 wsgi_intercept/_urllib3.py diff --git a/wsgi_intercept/_urllib3.py b/wsgi_intercept/_urllib3.py new file mode 100644 index 0000000..4661b43 --- /dev/null +++ b/wsgi_intercept/_urllib3.py @@ -0,0 +1,45 @@ +"""Common code of urllib3 and requests intercepts.""" + +import os +import sys + +from . import WSGI_HTTPConnection, WSGI_HTTPSConnection, wsgi_fake_socket + + +wsgi_fake_socket.settimeout = lambda self, timeout: None + + +def make_urllib3_override(HTTPConnectionPool, HTTPSConnectionPool, + HTTPConnection, HTTPSConnection): + + class HTTP_WSGIInterceptor(WSGI_HTTPConnection, HTTPConnection): + def __init__(self, *args, **kwargs): + if 'strict' in kwargs and sys.version_info > (3, 0): + kwargs.pop('strict') + WSGI_HTTPConnection.__init__(self, *args, **kwargs) + HTTPConnection.__init__(self, *args, **kwargs) + + + class HTTPS_WSGIInterceptor(WSGI_HTTPSConnection, HTTPSConnection): + is_verified = True + + def __init__(self, *args, **kwargs): + if 'strict' in kwargs and sys.version_info > (3, 0): + kwargs.pop('strict') + WSGI_HTTPSConnection.__init__(self, *args, **kwargs) + HTTPSConnection.__init__(self, *args, **kwargs) + + + def install(): + if 'http_proxy' in os.environ or 'https_proxy' in os.environ: + raise RuntimeError( + 'http_proxy or https_proxy set in environment, please unset') + HTTPConnectionPool.ConnectionCls = HTTP_WSGIInterceptor + HTTPSConnectionPool.ConnectionCls = HTTPS_WSGIInterceptor + + + def uninstall(): + HTTPConnectionPool.ConnectionCls = HTTPConnection + HTTPSConnectionPool.ConnectionCls = HTTPSConnection + + return install, uninstall diff --git a/wsgi_intercept/requests_intercept.py b/wsgi_intercept/requests_intercept.py index aff4ea2..5234b0e 100644 --- a/wsgi_intercept/requests_intercept.py +++ b/wsgi_intercept/requests_intercept.py @@ -2,45 +2,14 @@ `requests `_. """ -import os -import sys - -from . import WSGI_HTTPConnection, WSGI_HTTPSConnection, wsgi_fake_socket from requests.packages.urllib3.connectionpool import (HTTPConnectionPool, HTTPSConnectionPool) from requests.packages.urllib3.connection import (HTTPConnection, HTTPSConnection) +from ._urllib3 import make_urllib3_override -wsgi_fake_socket.settimeout = lambda self, timeout: None - - -class HTTP_WSGIInterceptor(WSGI_HTTPConnection, HTTPConnection): - def __init__(self, *args, **kwargs): - if 'strict' in kwargs and sys.version_info > (3, 0): - kwargs.pop('strict') - WSGI_HTTPConnection.__init__(self, *args, **kwargs) - HTTPConnection.__init__(self, *args, **kwargs) - - -class HTTPS_WSGIInterceptor(WSGI_HTTPSConnection, HTTPSConnection): - is_verified = True - - def __init__(self, *args, **kwargs): - if 'strict' in kwargs and sys.version_info > (3, 0): - kwargs.pop('strict') - WSGI_HTTPSConnection.__init__(self, *args, **kwargs) - HTTPSConnection.__init__(self, *args, **kwargs) - - -def install(): - if 'http_proxy' in os.environ or 'https_proxy' in os.environ: - raise RuntimeError( - 'http_proxy or https_proxy set in environment, please unset') - HTTPConnectionPool.ConnectionCls = HTTP_WSGIInterceptor - HTTPSConnectionPool.ConnectionCls = HTTPS_WSGIInterceptor - - -def uninstall(): - HTTPConnectionPool.ConnectionCls = HTTPConnection - HTTPSConnectionPool.ConnectionCls = HTTPSConnection +install, uninstall = make_urllib3_override(HTTPConnectionPool, + HTTPSConnectionPool, + HTTPConnection, + HTTPSConnection) diff --git a/wsgi_intercept/urllib3_intercept.py b/wsgi_intercept/urllib3_intercept.py index bb7b377..eff3997 100644 --- a/wsgi_intercept/urllib3_intercept.py +++ b/wsgi_intercept/urllib3_intercept.py @@ -2,45 +2,12 @@ `urllib3 `_. """ -import os -import sys - -# TODO: This is a a total dupe of requests. Need to remove -# duplication. -from . import WSGI_HTTPConnection, WSGI_HTTPSConnection, wsgi_fake_socket from urllib3.connectionpool import HTTPConnectionPool, HTTPSConnectionPool from urllib3.connection import HTTPConnection, HTTPSConnection +from ._urllib3 import make_urllib3_override -wsgi_fake_socket.settimeout = lambda self, timeout: None - - -class HTTP_WSGIInterceptor(WSGI_HTTPConnection, HTTPConnection): - def __init__(self, *args, **kwargs): - if 'strict' in kwargs and sys.version_info > (3, 0): - kwargs.pop('strict') - WSGI_HTTPConnection.__init__(self, *args, **kwargs) - HTTPConnection.__init__(self, *args, **kwargs) - - -class HTTPS_WSGIInterceptor(WSGI_HTTPSConnection, HTTPSConnection): - is_verified = True - - def __init__(self, *args, **kwargs): - if 'strict' in kwargs and sys.version_info > (3, 0): - kwargs.pop('strict') - WSGI_HTTPSConnection.__init__(self, *args, **kwargs) - HTTPSConnection.__init__(self, *args, **kwargs) - - -def install(): - if 'http_proxy' in os.environ or 'https_proxy' in os.environ: - raise RuntimeError( - 'http_proxy or https_proxy set in environment, please unset') - HTTPConnectionPool.ConnectionCls = HTTP_WSGIInterceptor - HTTPSConnectionPool.ConnectionCls = HTTPS_WSGIInterceptor - - -def uninstall(): - HTTPConnectionPool.ConnectionCls = HTTPConnection - HTTPSConnectionPool.ConnectionCls = HTTPSConnection +install, uninstall = make_urllib3_override(HTTPConnectionPool, + HTTPSConnectionPool, + HTTPConnection, + HTTPSConnection) From 06387f3ddafaf7de035104daf8344b1b1070e776 Mon Sep 17 00:00:00 2001 From: Chris Dent Date: Fri, 18 Mar 2016 14:22:21 +0000 Subject: [PATCH 5/5] Add urllib3 to the docs Update the README to reflect the modern condition. --- docs/index.rst | 1 + docs/urllib3.rst | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 docs/urllib3.rst diff --git a/docs/index.rst b/docs/index.rst index 6058670..5271ed2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -15,6 +15,7 @@ Examples http_client httplib2 requests + urllib3 urllib diff --git a/docs/urllib3.rst b/docs/urllib3.rst new file mode 100644 index 0000000..a3b60fc --- /dev/null +++ b/docs/urllib3.rst @@ -0,0 +1,32 @@ +urllib3_intercept +================== + +.. automodule:: wsgi_intercept.urllib3_intercept + + +Example: + +.. testcode:: + + import urllib3 + from wsgi_intercept import urllib3_intercept, add_wsgi_intercept + + pool = urllib3.PoolManager() + + + def app(environ, start_response): + start_response('200 OK', [('Content-Type', 'text/plain')]) + return [b'Whee'] + + + def make_app(): + return app + + + host, port = 'localhost', 80 + url = 'http://{0}:{1}/'.format(host, port) + urllib3_intercept.install() + add_wsgi_intercept(host, port, make_app) + resp = pool.requests('GET', url) + assert resp.data == b'Whee' + urllib3_intercept.uninstall()