Add a glance fetcher

Add a default fetcher that obtains copies of images via glance api v2.

Also refactor glance client code to be re-usable in cachemonkey.glance

cachemonkey/cacher.py

@@ -13,6 +13,8 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import os
+
 from oslo.config import cfg
 
 from cachemonkey.openstack.common import importutils
@@ -25,6 +27,12 @@ opts = [
     cfg.StrOpt('lister_class',
                default='cachemonkey.lister.glance.GlanceLister',
                help='Class to determine which images get pre-cached'),
+    cfg.StrOpt('fetcher_class',
+               default='cachemonkey.fetcher.glance.GlanceFetcher',
+               help='Class to determine how to fetch images.'),
+    cfg.StrOpt('data_dir', default='/var/lib/cachemonkey',
+               help='Directory containing image data'),
+
 ]
 
 CONF.register_opts(opts, group='cachemonkey')
@@ -34,8 +42,26 @@ class Cacher(object):
 
     def __init__(self):
         self.lister = importutils.import_object(CONF.cachemonkey.lister_class)
+        self.fetcher = importutils.import_object(
+            CONF.cachemonkey.fetcher_class)
 
     def cache(self):
         images = self.lister.images()
         for image in images:
-            LOG.debug(image)
+            self._get(image)
+            # TODO(belliott) distribute to hosts
+
+    def _get(self, image):
+        # first see if the image was previously downloaded
+        d = os.path.join(CONF.cachemonkey.data_dir)
+        if not os.path.exists(d):
+            os.makedirs(d)
+
+        filename = os.path.join(d, image['id'])
+        if os.path.exists(filename):
+            LOG.debug("Image %s was previously downloaded." % image['id'])
+            # TODO(belliott) also verify checksum
+        else:
+            self.fetcher.fetch(image, filename)
+
+        return filename

cachemonkey/fetcher/glance.py

@@ -0,0 +1,38 @@
+# Copyright 2014 Rackspace Hosting
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from cachemonkey import glance
+from cachemonkey.openstack.common import log as logging
+
+LOG = logging.getLogger(__name__)
+
+
+class GlanceFetcher(object):
+    """Fetch image data via Glance."""
+
+    def __init__(self):
+        self.client = glance.Client()
+
+    def fetch(self, image, filename):
+        LOG.debug('Fetching image %s' % image['id'])
+
+        # glance client returns a iterator over the response
+        # downloads 64KB at a time.
+        resp = self.client.images.data(image['id'], do_checksum=True)
+
+        f = open(filename, 'wb')
+        for chunk in resp:
+            f.write(chunk)
+        f.close()

cachemonkey/glance.py

@@ -0,0 +1,77 @@
+# Copyright 2014 Rackspace Hosting
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import glanceclient
+from oslo.config import cfg
+
+from cachemonkey import auth
+from cachemonkey.openstack.common import log as logging
+
+LOG = logging.getLogger(__name__)
+CONF = cfg.CONF
+
+opts = [
+    cfg.ListOpt('endpoints', help='Glance endpoint (override catalog)'),
+    cfg.StrOpt('scheme', help='Glance endpoint URL scheme', default='http')
+]
+CONF.register_opts(opts, group='glance')
+
+
+class Client(object):
+    """Glance client wrapper that handles auth and config."""
+
+    def __init__(self):
+        self.authclient = auth.Client()
+        self.authclient.auth()
+
+        self.endpoints = self._endpoints()
+        self.next_ = 0
+
+    def __getattr__(self, key):
+        client = self._client()
+        return getattr(client, key)
+
+    def _client(self):
+        # pick a glance endpoint via round-robin
+        endpoint = self.endpoints[self.next_]
+        self.next_ += 1
+
+        api_version = 2
+        token = self.authclient.token
+        return glanceclient.Client(api_version, endpoint, token=token)
+
+    def _endpoints(self):
+        # prefer endpoints provided via config
+        if CONF.glance.endpoints:
+            endpoints = CONF.glance.endpoints
+        else:
+            # fallback to getting a glance endpoint from the service catalog
+            catalog = self.authclient.catalog
+            image_endpoints = catalog.get_endpoints()['image']
+
+            LOG.debug('Image endpoints: %s' % image_endpoints)
+            if len(image_endpoints) > 1:
+                LOG.warn('Expected a single image endpoint, but there are %d' %
+                         len(image_endpoints))
+
+            endpoint = image_endpoints[0]['publicURL']
+            endpoints = [endpoint]
+
+        if endpoints[0].find('://') == -1:
+            # prepend a scheme to each endpoint
+            scheme = CONF.glance.scheme
+            endpoints = map(lambda e: "%s://%s" % (scheme, e), endpoints)
+
+        return endpoints

cachemonkey/lister/glance.py

@@ -13,48 +13,19 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import glanceclient
-from oslo.config import cfg
-
-from cachemonkey import auth
+from cachemonkey import glance
 from cachemonkey.openstack.common import log as logging
 
 LOG = logging.getLogger(__name__)
-CONF = cfg.CONF
-
-opt = cfg.StrOpt('endpoint', help='Glance endpoint (override catalog)')
-CONF.register_opt(opt, group='glance')
 
 
 class GlanceLister(object):
 
     def __init__(self):
-        self.authclient = auth.Client()
-        self.authclient.auth()
-        endpoint = self._endpoint()
-
-        api_version = 2
-        token = self.authclient.token
-        self.client = glanceclient.Client(api_version, endpoint, token=token)
+        self.client = glance.Client()
 
     def images(self):
         LOG.debug('Fetching image list')
         # list images to be cached.
         images = self.client.images.list()
         return images
-
-    def _endpoint(self):
-        # if an endpoint is provided via config, prefer that
-        if CONF.glance.endpoint:
-            return CONF.glance.endpoint
-
-        # fallback to getting the glance endpoint from the service catalog
-        catalog = self.authclient.catalog
-        image_endpoints = catalog.get_endpoints()['image']
-
-        LOG.debug('Image endpoints: %s' % image_endpoints)
-        if len(image_endpoints) > 1:
-            LOG.warn('Expected a single image endpoint, but there are %d' %
-                     len(image_endpoints))
-
-        return image_endpoints[0]['publicURL']