diff --git a/docs/index.rst b/docs/index.rst index 6058670..5271ed2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -15,6 +15,7 @@ Examples http_client httplib2 requests + urllib3 urllib diff --git a/docs/urllib3.rst b/docs/urllib3.rst new file mode 100644 index 0000000..a3b60fc --- /dev/null +++ b/docs/urllib3.rst @@ -0,0 +1,32 @@ +urllib3_intercept +================== + +.. automodule:: wsgi_intercept.urllib3_intercept + + +Example: + +.. testcode:: + + import urllib3 + from wsgi_intercept import urllib3_intercept, add_wsgi_intercept + + pool = urllib3.PoolManager() + + + def app(environ, start_response): + start_response('200 OK', [('Content-Type', 'text/plain')]) + return [b'Whee'] + + + def make_app(): + return app + + + host, port = 'localhost', 80 + url = 'http://{0}:{1}/'.format(host, port) + urllib3_intercept.install() + add_wsgi_intercept(host, port, make_app) + resp = pool.requests('GET', url) + assert resp.data == b'Whee' + urllib3_intercept.uninstall() diff --git a/setup.py b/setup.py index 53037cd..3925630 100644 --- a/setup.py +++ b/setup.py @@ -38,6 +38,7 @@ META = { 'pytest>=2.4', 'httplib2', 'requests>=2.0.1', + 'urllib3', ], }, } diff --git a/test/test_interceptor.py b/test/test_interceptor.py index ca37ebd..66e0881 100644 --- a/test/test_interceptor.py +++ b/test/test_interceptor.py @@ -9,6 +9,7 @@ from uuid import uuid4 import py.test import requests +import urllib3 from httplib2 import Http, ServerNotFoundError from six.moves import http_client from six.moves.urllib.request import urlopen @@ -16,9 +17,10 @@ from six.moves.urllib.error import URLError from wsgi_intercept.interceptor import ( Interceptor, HttpClientInterceptor, Httplib2Interceptor, - RequestsInterceptor, UrllibInterceptor) + RequestsInterceptor, UrllibInterceptor, Urllib3Interceptor) from .wsgi_app import simple_app +httppool = urllib3.PoolManager() def app(): return simple_app @@ -178,6 +180,41 @@ def test_requests_in_out(): requests.get(url) +# urllib3 + +def test_urllib3_interceptor_host(): + hostname = str(uuid4()) + port = 9999 + with Urllib3Interceptor(app=app, host=hostname, port=port) as url: + response = httppool.request('GET', url) + assert response.status == 200 + assert 'WSGI intercept successful!' in str(response.data) + + +def test_urllib3_interceptor_url(): + hostname = str(uuid4()) + port = 9999 + url = 'http://%s:%s/' % (hostname, port) + with Urllib3Interceptor(app=app, url=url) as target_url: + response = httppool.request('GET', target_url) + assert response.status == 200 + assert 'WSGI intercept successful!' in str(response.data) + + +def test_urllib3_in_out(): + hostname = str(uuid4()) + port = 9999 + url = 'http://%s:%s/' % (hostname, port) + with Urllib3Interceptor(app=app, url=url) as target_url: + response = httppool.request('GET', target_url) + assert response.status == 200 + assert 'WSGI intercept successful!' in str(response.data) + + # outside the context manager the intercept does not work + with py.test.raises(urllib3.exceptions.MaxRetryError): + httppool.request('GET', url) + + # urllib def test_urllib_interceptor_host(): diff --git a/test/test_urllib3.py b/test/test_urllib3.py new file mode 100644 index 0000000..8206d5e --- /dev/null +++ b/test/test_urllib3.py @@ -0,0 +1,90 @@ +import os +import py.test +import socket +from wsgi_intercept import urllib3_intercept, WSGIAppError +from test import wsgi_app +from test.install import installer_class, skipnetwork +import urllib3 + +HOST = 'some_hopefully_nonexistant_domain' + +InstalledApp = installer_class(urllib3_intercept) +http = urllib3.PoolManager() + + +def test_http(): + with InstalledApp(wsgi_app.simple_app, host=HOST, port=80) as app: + resp = http.request('GET', 'http://some_hopefully_nonexistant_domain:80/') + assert resp.data == b'WSGI intercept successful!\n' + assert app.success() + + +def test_http_default_port(): + with InstalledApp(wsgi_app.simple_app, host=HOST, port=80) as app: + resp = http.request('GET', 'http://some_hopefully_nonexistant_domain/') + assert resp.data == b'WSGI intercept successful!\n' + assert app.success() + + +def test_http_other_port(): + with InstalledApp(wsgi_app.simple_app, host=HOST, port=8080) as app: + resp = http.request('GET', 'http://some_hopefully_nonexistant_domain:8080/') + assert resp.data == b'WSGI intercept successful!\n' + assert app.success() + environ = app.get_internals() + assert environ['wsgi.url_scheme'] == 'http' + + +def test_bogus_domain(): + with InstalledApp(wsgi_app.simple_app, host=HOST, port=80): + py.test.raises( + urllib3.exceptions.MaxRetryError, + 'http.request("GET", "http://_nonexistant_domain_")') + + +def test_proxy_handling(): + with py.test.raises(RuntimeError) as exc: + with InstalledApp(wsgi_app.simple_app, host=HOST, port=80, + proxy='some_proxy.com:1234'): + http.request('GET', 'http://some_hopefully_nonexistant_domain:80/') + assert 'http_proxy or https_proxy set in environment' in str(exc.value) + # We need to do this by hand because the exception was raised + # during the entry of the context manager, so the exit handler + # wasn't reached. + del os.environ['http_proxy'] + + +def test_https(): + with InstalledApp(wsgi_app.simple_app, host=HOST, port=443) as app: + resp = http.request('GET', 'https://some_hopefully_nonexistant_domain:443/') + assert resp.data == b'WSGI intercept successful!\n' + assert app.success() + + +def test_https_default_port(): + with InstalledApp(wsgi_app.simple_app, host=HOST, port=443) as app: + resp = http.request('GET', 'https://some_hopefully_nonexistant_domain/') + assert resp.data == b'WSGI intercept successful!\n' + assert app.success() + environ = app.get_internals() + assert environ['wsgi.url_scheme'] == 'https' + + +def test_app_error(): + with InstalledApp(wsgi_app.raises_app, host=HOST, port=80): + with py.test.raises(WSGIAppError): + http.request('GET', 'http://some_hopefully_nonexistant_domain/') + + +@skipnetwork +def test_http_not_intercepted(): + with InstalledApp(wsgi_app.raises_app, host=HOST, port=80): + resp = http.request('GET', 'http://google.com') + assert resp.status >= 200 and resp.status < 300 + + +@skipnetwork +def test_https_not_intercepted(): + with InstalledApp(wsgi_app.raises_app, host=HOST, port=80): + resp = http.request('GET', 'https://google.com') + assert resp.status >= 200 and resp.status < 300 diff --git a/wsgi_intercept/_urllib3.py b/wsgi_intercept/_urllib3.py new file mode 100644 index 0000000..4661b43 --- /dev/null +++ b/wsgi_intercept/_urllib3.py @@ -0,0 +1,45 @@ +"""Common code of urllib3 and requests intercepts.""" + +import os +import sys + +from . import WSGI_HTTPConnection, WSGI_HTTPSConnection, wsgi_fake_socket + + +wsgi_fake_socket.settimeout = lambda self, timeout: None + + +def make_urllib3_override(HTTPConnectionPool, HTTPSConnectionPool, + HTTPConnection, HTTPSConnection): + + class HTTP_WSGIInterceptor(WSGI_HTTPConnection, HTTPConnection): + def __init__(self, *args, **kwargs): + if 'strict' in kwargs and sys.version_info > (3, 0): + kwargs.pop('strict') + WSGI_HTTPConnection.__init__(self, *args, **kwargs) + HTTPConnection.__init__(self, *args, **kwargs) + + + class HTTPS_WSGIInterceptor(WSGI_HTTPSConnection, HTTPSConnection): + is_verified = True + + def __init__(self, *args, **kwargs): + if 'strict' in kwargs and sys.version_info > (3, 0): + kwargs.pop('strict') + WSGI_HTTPSConnection.__init__(self, *args, **kwargs) + HTTPSConnection.__init__(self, *args, **kwargs) + + + def install(): + if 'http_proxy' in os.environ or 'https_proxy' in os.environ: + raise RuntimeError( + 'http_proxy or https_proxy set in environment, please unset') + HTTPConnectionPool.ConnectionCls = HTTP_WSGIInterceptor + HTTPSConnectionPool.ConnectionCls = HTTPS_WSGIInterceptor + + + def uninstall(): + HTTPConnectionPool.ConnectionCls = HTTPConnection + HTTPSConnectionPool.ConnectionCls = HTTPSConnection + + return install, uninstall diff --git a/wsgi_intercept/interceptor.py b/wsgi_intercept/interceptor.py index fd48c1a..3daea37 100644 --- a/wsgi_intercept/interceptor.py +++ b/wsgi_intercept/interceptor.py @@ -109,6 +109,12 @@ class RequestsInterceptor(Interceptor): MODULE_NAME = 'requests_intercept' +class Urllib3Interceptor(Interceptor): + """Interceptor for requests.""" + + MODULE_NAME = 'urllib3_intercept' + + class UrllibInterceptor(Interceptor): """Interceptor for urllib2 and urllib.request.""" diff --git a/wsgi_intercept/requests_intercept.py b/wsgi_intercept/requests_intercept.py index aff4ea2..5234b0e 100644 --- a/wsgi_intercept/requests_intercept.py +++ b/wsgi_intercept/requests_intercept.py @@ -2,45 +2,14 @@ `requests `_. """ -import os -import sys - -from . import WSGI_HTTPConnection, WSGI_HTTPSConnection, wsgi_fake_socket from requests.packages.urllib3.connectionpool import (HTTPConnectionPool, HTTPSConnectionPool) from requests.packages.urllib3.connection import (HTTPConnection, HTTPSConnection) +from ._urllib3 import make_urllib3_override -wsgi_fake_socket.settimeout = lambda self, timeout: None - - -class HTTP_WSGIInterceptor(WSGI_HTTPConnection, HTTPConnection): - def __init__(self, *args, **kwargs): - if 'strict' in kwargs and sys.version_info > (3, 0): - kwargs.pop('strict') - WSGI_HTTPConnection.__init__(self, *args, **kwargs) - HTTPConnection.__init__(self, *args, **kwargs) - - -class HTTPS_WSGIInterceptor(WSGI_HTTPSConnection, HTTPSConnection): - is_verified = True - - def __init__(self, *args, **kwargs): - if 'strict' in kwargs and sys.version_info > (3, 0): - kwargs.pop('strict') - WSGI_HTTPSConnection.__init__(self, *args, **kwargs) - HTTPSConnection.__init__(self, *args, **kwargs) - - -def install(): - if 'http_proxy' in os.environ or 'https_proxy' in os.environ: - raise RuntimeError( - 'http_proxy or https_proxy set in environment, please unset') - HTTPConnectionPool.ConnectionCls = HTTP_WSGIInterceptor - HTTPSConnectionPool.ConnectionCls = HTTPS_WSGIInterceptor - - -def uninstall(): - HTTPConnectionPool.ConnectionCls = HTTPConnection - HTTPSConnectionPool.ConnectionCls = HTTPSConnection +install, uninstall = make_urllib3_override(HTTPConnectionPool, + HTTPSConnectionPool, + HTTPConnection, + HTTPSConnection) diff --git a/wsgi_intercept/urllib3_intercept.py b/wsgi_intercept/urllib3_intercept.py new file mode 100644 index 0000000..eff3997 --- /dev/null +++ b/wsgi_intercept/urllib3_intercept.py @@ -0,0 +1,13 @@ +"""Intercept HTTP connections that use +`urllib3 `_. +""" + +from urllib3.connectionpool import HTTPConnectionPool, HTTPSConnectionPool +from urllib3.connection import HTTPConnection, HTTPSConnection +from ._urllib3 import make_urllib3_override + + +install, uninstall = make_urllib3_override(HTTPConnectionPool, + HTTPSConnectionPool, + HTTPConnection, + HTTPSConnection)