URI filtering for web-download

Implement URI filtering to prevent port scanning with the web-download
Image import method.

Closes-Bug: #1748512

Change-Id: Ide5ace8979bb12239c99a312747b3151c1e64ce8

etc/glance-image-import.conf.sample

@@ -32,6 +32,175 @@
 #image_import_plugins = [no_op]
 
 
+[import_filtering_opts]
+
+#
+# From glance
+#
+
+#
+# Specify the allowed url schemes for web-download.
+#
+# This option provides whitelisting for uri schemes that web-download import
+# method will be using. Whitelisting is always priority and ignores any
+# blacklisting of the schemes but obeys host and port filtering.
+#
+# For example: If scheme blacklisting contains 'http' and whitelist contains
+# ['http', 'https'] the whitelist is obeyed on http://example.com but any
+# other scheme like ftp://example.com is blocked even it's not blacklisted.
+#
+# Possible values:
+#     * List containing normalized url schemes as they are returned from
+#     urllib.parse. For example ['ftp','https']
+#
+# Related options:
+#     * disallowed_schemes
+#     * allowed_hosts
+#     * disallowed_hosts
+#     * allowed_ports
+#     * disallowed_ports
+#
+#  (list value)
+#allowed_schemes = http,https
+
+#
+# Specify the blacklisted url schemes for web-download.
+#
+# This option provides blacklisting for uri schemes that web-download import
+# method will be using. Whitelisting is always priority and ignores any
+# blacklisting of the schemes but obeys host and port filtering. Blacklisting
+# can be used to prevent specific scheme to be used when whitelisting is not
+# in use.
+#
+# For example: If scheme blacklisting contains 'http' and whitelist contains
+# ['http', 'https'] the whitelist is obeyed on http://example.com but any
+# other scheme like ftp://example.com is blocked even it's not blacklisted.
+#
+# Possible values:
+#     * List containing normalized url schemes as they are returned from
+#     urllib.parse. For example ['ftp','https']
+#     * By default the list is empty
+#
+# Related options:
+#     * allowed_schemes
+#     * allowed_hosts
+#     * disallowed_hosts
+#     * allowed_ports
+#     * disallowed_ports
+#
+#  (list value)
+#disallowed_schemes =
+
+#
+# Specify the allowed target hosts for web-download.
+#
+# This option provides whitelisting for hosts that web-download import
+# method will be using. Whitelisting is always priority and ignores any
+# blacklisting of the hosts but obeys scheme and port filtering.
+#
+# For example: If scheme blacklisting contains 'http' and whitelist contains
+# ['http', 'https'] the whitelist is obeyed on http://example.com but any
+# other scheme like ftp://example.com is blocked even it's not blacklisted.
+# Same way the whitelisted example.com is only obeyed on the allowed schemes
+# and or ports. Whitelisting of the host does not allow all schemes and ports
+# accessed.
+#
+# Possible values:
+#     * List containing normalized hostname or ip like it would be returned
+#     in the urllib.parse netloc without the port
+#     * By default the list is empty
+#
+# Related options:
+#     * allowed_schemes
+#     * disallowed_schemes
+#     * disallowed_hosts
+#     * allowed_ports
+#     * disallowed_ports
+#
+#  (list value)
+#allowed_hosts =
+
+#
+# Specify the blacklisted hosts for web-download.
+#
+# This option provides blacklisting for hosts that web-download import
+# method will be using. Whitelisting is always priority and ignores any
+# blacklisting but obeys scheme and port filtering.
+#
+# For example: If scheme blacklisting contains 'http' and whitelist contains
+# ['http', 'https'] the whitelist is obeyed on http://example.com but any
+# other scheme like ftp://example.com is blocked even it's not blacklisted.
+# The blacklisted example.com is obeyed on any url pointing to that host
+# regardless of what their scheme or port is.
+#
+# Possible values:
+#     * List containing normalized hostname or ip like it would be returned
+#     in the urllib.parse netloc without the port
+#     * By default the list is empty
+#
+# Related options:
+#     * allowed_schemes
+#     * disallowed_schemes
+#     * allowed_hosts
+#     * allowed_ports
+#     * disallowed_ports
+#
+#  (list value)
+#disallowed_hosts =
+
+#
+# Specify the allowed ports for web-download.
+#
+# This option provides whitelisting for uri ports that web-download import
+# method will be using. Whitelisting is always priority and ignores any
+# blacklisting of the ports but obeys host and scheme filtering.
+#
+# For example: If scheme blacklisting contains '80' and whitelist contains
+# ['80', '443'] the whitelist is obeyed on http://example.com:80 but any
+# other port like ftp://example.com:21 is blocked even it's not blacklisted.
+#
+# Possible values:
+#     * List containing ports as they are returned from urllib.parse netloc
+#     field. For example ['80','443']
+#
+# Related options:
+#     * allowed_schemes
+#     * disallowed_schemes
+#     * allowed_hosts
+#     * disallowed_hosts
+#     * disallowed_ports
+#  (list value)
+#allowed_ports = 80,443
+
+#
+# Specify the disallowed ports for web-download.
+#
+# This option provides blacklisting for uri ports that web-download import
+# method will be using. Whitelisting is always priority and ignores any
+# blacklisting of the ports but obeys host and scheme filtering.
+#
+# For example: If scheme blacklisting contains '80' and whitelist contains
+# ['80', '443'] the whitelist is obeyed on http://example.com:80 but any
+# other port like ftp://example.com:21 is blocked even it's not blacklisted.
+# If no whitelisting is defined any scheme and host combination is disallowed
+# for the blacklisted port.
+#
+# Possible values:
+#     * List containing ports as they are returned from urllib.parse netloc
+#     field. For example ['80','443']
+#     * By default this list is empty.
+#
+# Related options:
+#     * allowed_schemes
+#     * disallowed_schemes
+#     * allowed_hosts
+#     * disallowed_hosts
+#     * allowed_ports
+#
+#  (list value)
+#disallowed_ports =
+
+
 [inject_metadata_properties]
 
 #

glance/api/v2/images.py

@@ -97,6 +97,16 @@ class ImagesController(object):
         task_input = {'image_id': image_id,
                       'import_req': body}
 
+        import_method = body.get('method').get('name')
+        uri = body.get('method').get('uri')
+        if (import_method == 'web-download' and
+           not utils.validate_import_uri(uri)):
+                LOG.debug("URI for web-download does not pass filtering: %s" %
+                          uri)
+                msg = (_("URI for web-download does not pass filtering: %s") %
+                       uri)
+                raise webob.exc.HTTPBadRequest(explanation=msg)
+
         try:
             import_task = task_factory.new_task(task_type='api_image_import',
                                                 owner=req.context.owner,

glance/async/flows/_internal_plugins/web_download.py

@@ -29,6 +29,187 @@ LOG = logging.getLogger(__name__)
 CONF = cfg.CONF
 
 
+import_filtering_opts = [
+
+    cfg.ListOpt('allowed_schemes',
+                item_type=cfg.types.String(quotes=True),
+                bounds=True,
+                default=['http', 'https'],
+                help=_("""
+Specify the allowed url schemes for web-download.
+
+This option provides whitelisting for uri schemes that web-download import
+method will be using. Whitelisting is always priority and ignores any
+blacklisting of the schemes but obeys host and port filtering.
+
+For example: If scheme blacklisting contains 'http' and whitelist contains
+['http', 'https'] the whitelist is obeyed on http://example.com but any
+other scheme like ftp://example.com is blocked even it's not blacklisted.
+
+Possible values:
+    * List containing normalized url schemes as they are returned from
+    urllib.parse. For example ['ftp','https']
+
+Related options:
+    * disallowed_schemes
+    * allowed_hosts
+    * disallowed_hosts
+    * allowed_ports
+    * disallowed_ports
+
+""")),
+    cfg.ListOpt('disallowed_schemes',
+                item_type=cfg.types.String(quotes=True),
+                bounds=True,
+                default=[],
+                help=_("""
+Specify the blacklisted url schemes for web-download.
+
+This option provides blacklisting for uri schemes that web-download import
+method will be using. Whitelisting is always priority and ignores any
+blacklisting of the schemes but obeys host and port filtering. Blacklisting
+can be used to prevent specific scheme to be used when whitelisting is not
+in use.
+
+For example: If scheme blacklisting contains 'http' and whitelist contains
+['http', 'https'] the whitelist is obeyed on http://example.com but any
+other scheme like ftp://example.com is blocked even it's not blacklisted.
+
+Possible values:
+    * List containing normalized url schemes as they are returned from
+    urllib.parse. For example ['ftp','https']
+    * By default the list is empty
+
+Related options:
+    * allowed_schemes
+    * allowed_hosts
+    * disallowed_hosts
+    * allowed_ports
+    * disallowed_ports
+
+""")),
+    cfg.ListOpt('allowed_hosts',
+                item_type=cfg.types.HostAddress(),
+                bounds=True,
+                default=[],
+                help=_("""
+Specify the allowed target hosts for web-download.
+
+This option provides whitelisting for hosts that web-download import
+method will be using. Whitelisting is always priority and ignores any
+blacklisting of the hosts but obeys scheme and port filtering.
+
+For example: If scheme blacklisting contains 'http' and whitelist contains
+['http', 'https'] the whitelist is obeyed on http://example.com but any
+other scheme like ftp://example.com is blocked even it's not blacklisted.
+Same way the whitelisted example.com is only obeyed on the allowed schemes
+and or ports. Whitelisting of the host does not allow all schemes and ports
+accessed.
+
+Possible values:
+    * List containing normalized hostname or ip like it would be returned
+    in the urllib.parse netloc without the port
+    * By default the list is empty
+
+Related options:
+    * allowed_schemes
+    * disallowed_schemes
+    * disallowed_hosts
+    * allowed_ports
+    * disallowed_ports
+
+""")),
+    cfg.ListOpt('disallowed_hosts',
+                item_type=cfg.types.HostAddress(),
+                bounds=True,
+                default=[],
+                help=_("""
+Specify the blacklisted hosts for web-download.
+
+This option provides blacklisting for hosts that web-download import
+method will be using. Whitelisting is always priority and ignores any
+blacklisting but obeys scheme and port filtering.
+
+For example: If scheme blacklisting contains 'http' and whitelist contains
+['http', 'https'] the whitelist is obeyed on http://example.com but any
+other scheme like ftp://example.com is blocked even it's not blacklisted.
+The blacklisted example.com is obeyed on any url pointing to that host
+regardless of what their scheme or port is.
+
+Possible values:
+    * List containing normalized hostname or ip like it would be returned
+    in the urllib.parse netloc without the port
+    * By default the list is empty
+
+Related options:
+    * allowed_schemes
+    * disallowed_schemes
+    * allowed_hosts
+    * allowed_ports
+    * disallowed_ports
+
+""")),
+    cfg.ListOpt('allowed_ports',
+                item_type=cfg.types.Integer(min=1, max=65535),
+                bounds=True,
+                default=[80, 443],
+                help=_("""
+Specify the allowed ports for web-download.
+
+This option provides whitelisting for uri ports that web-download import
+method will be using. Whitelisting is always priority and ignores any
+blacklisting of the ports but obeys host and scheme filtering.
+
+For example: If scheme blacklisting contains '80' and whitelist contains
+['80', '443'] the whitelist is obeyed on http://example.com:80 but any
+other port like ftp://example.com:21 is blocked even it's not blacklisted.
+
+Possible values:
+    * List containing ports as they are returned from urllib.parse netloc
+    field. For example ['80','443']
+
+Related options:
+    * allowed_schemes
+    * disallowed_schemes
+    * allowed_hosts
+    * disallowed_hosts
+    * disallowed_ports
+""")),
+    cfg.ListOpt('disallowed_ports',
+                item_type=cfg.types.Integer(min=1, max=65535),
+                bounds=True,
+                default=[],
+                help=_("""
+Specify the disallowed ports for web-download.
+
+This option provides blacklisting for uri ports that web-download import
+method will be using. Whitelisting is always priority and ignores any
+blacklisting of the ports but obeys host and scheme filtering.
+
+For example: If scheme blacklisting contains '80' and whitelist contains
+['80', '443'] the whitelist is obeyed on http://example.com:80 but any
+other port like ftp://example.com:21 is blocked even it's not blacklisted.
+If no whitelisting is defined any scheme and host combination is disallowed
+for the blacklisted port.
+
+Possible values:
+    * List containing ports as they are returned from urllib.parse netloc
+    field. For example ['80','443']
+    * By default this list is empty.
+
+Related options:
+    * allowed_schemes
+    * disallowed_schemes
+    * allowed_hosts
+    * disallowed_hosts
+    * allowed_ports
+
+""")),
+]
+
+CONF.register_opts(import_filtering_opts, group='import_filtering_opts')
+
+
 class _WebDownload(task.Task):
 
     default_provides = 'file_uri'

glance/common/utils.py

@@ -41,6 +41,7 @@ from oslo_utils import excutils
 from oslo_utils import netutils
 from oslo_utils import strutils
 import six
+from six.moves import urllib
 from webob import exc
 
 from glance.common import exception
@@ -127,6 +128,54 @@ def cooperative_read(fd):
 MAX_COOP_READER_BUFFER_SIZE = 134217728  # 128M seems like a sane buffer limit
 
 
+def validate_import_uri(uri):
+    """Validate requested uri for Image Import web-download.
+
+    :param uri: target uri to be validated
+    """
+
+    if not uri:
+        return False
+
+    parsed_uri = urllib.parse.urlparse(uri)
+    scheme = parsed_uri.scheme
+    host = parsed_uri.hostname
+    port = parsed_uri.port
+    wl_schemes = CONF.import_filtering_opts.allowed_schemes
+    bl_schemes = CONF.import_filtering_opts.disallowed_schemes
+    wl_hosts = CONF.import_filtering_opts.allowed_hosts
+    bl_hosts = CONF.import_filtering_opts.disallowed_hosts
+    wl_ports = CONF.import_filtering_opts.allowed_ports
+    bl_ports = CONF.import_filtering_opts.disallowed_ports
+
+    # NOTE(jokke): Checking if both allowed and disallowed are defined and
+    # logging it to inform only allowed will be obeyed.
+    if wl_schemes and bl_schemes:
+        bl_schemes = []
+        LOG.debug("Both allowed and disallowed schemes has been configured."
+                  "Will only process allowed list.")
+    if wl_hosts and bl_hosts:
+        bl_hosts = []
+        LOG.debug("Both allowed and disallowed hosts has been configured."
+                  "Will only process allowed list.")
+    if wl_ports and bl_ports:
+        bl_ports = []
+        LOG.debug("Both allowed and disallowed ports has been configured."
+                  "Will only process allowed list.")
+
+    if not scheme or ((wl_schemes and scheme not in wl_schemes) or
+                      parsed_uri.scheme in bl_schemes):
+        return False
+    if not host or ((wl_hosts and host not in wl_hosts) or
+                    host in bl_hosts):
+        return False
+    if port and ((wl_ports and port not in wl_ports) or
+                 port in bl_ports):
+        return False
+
+    return True
+
+
 class CooperativeReader(object):
     """
     An eventlet thread friendly class for reading in image data.

glance/opts.py

@@ -28,6 +28,7 @@ from osprofiler import opts as profiler
 
 import glance.api.middleware.context
 import glance.api.versions
+import glance.async.flows._internal_plugins.web_download
 import glance.async.flows.api_image_import
 import glance.async.flows.convert
 from glance.async.flows.plugins import plugin_opts
@@ -109,6 +110,8 @@ _manage_opts = [
 ]
 _image_import_opts = [
     ('image_import_opts', glance.async.flows.api_image_import.api_import_opts),
+    ('import_filtering_opts',
+     glance.async.flows._internal_plugins.web_download.import_filtering_opts),
 ]
 
 

glance/tests/unit/v2/test_images_resource.py

@@ -610,7 +610,7 @@ class TestImagesController(base.IsolatedUnitTest):
                                side_effect=exception.Conflict):
             self.assertRaises(webob.exc.HTTPConflict,
                               self.controller.import_image, request, UUID4,
-                              {})
+                              {'method': {'name': 'glance-direct'}})
 
     def test_image_import_raises_conflict_for_invalid_status_change(self):
         request = unit_test_utils.get_fake_request()
@@ -623,7 +623,7 @@ class TestImagesController(base.IsolatedUnitTest):
                 side_effect=exception.InvalidImageStatusTransition):
             self.assertRaises(webob.exc.HTTPConflict,
                               self.controller.import_image, request, UUID4,
-                              {})
+                              {'method': {'name': 'glance-direct'}})
 
     def test_image_import_raises_bad_request(self):
         request = unit_test_utils.get_fake_request()
@@ -635,7 +635,7 @@ class TestImagesController(base.IsolatedUnitTest):
                                side_effect=ValueError):
             self.assertRaises(webob.exc.HTTPBadRequest,
                               self.controller.import_image, request, UUID4,
-                              {})
+                              {'method': {'name': 'glance-direct'}})
 
     def test_create(self):
         request = unit_test_utils.get_fake_request()
@@ -2246,7 +2246,8 @@ class TestImagesController(base.IsolatedUnitTest):
     def test_image_import(self):
         request = unit_test_utils.get_fake_request()
         output = self.controller.import_image(request, UUID4,
-                                              {})
+                                              {'method': {'name':
+                                                          'glance-direct'}})
         self.assertEqual(UUID4, output)
 
     def test_image_import_not_allowed(self):
@@ -2256,7 +2257,8 @@ class TestImagesController(base.IsolatedUnitTest):
         request.context.tenant = None
         self.assertRaises(webob.exc.HTTPForbidden,
                           self.controller.import_image,
-                          request, UUID4, {})
+                          request, UUID4, {'method': {'name':
+                                                      'glance-direct'}})
 
 
 class TestImagesControllerPolicies(base.IsolatedUnitTest):