diff --git a/devstack/gluster-functions.sh b/devstack/gluster-functions.sh index ed66dc9..a96b9d8 100755 --- a/devstack/gluster-functions.sh +++ b/devstack/gluster-functions.sh @@ -342,9 +342,48 @@ function _configure_manila_glusterfs_native { iniset $MANILA_CONF DEFAULT enabled_share_backends $group_name } +function _setup_rootssh { + mkdir -p "$HOME"/.ssh + chmod 700 "$HOME"/.ssh + sudo mkdir -p /root/.ssh + sudo chmod 700 /root/.ssh + yes n | ssh-keygen -f "$HOME"/.ssh/id_rsa -N '' + sudo sh -c "cat >> /root/.ssh/authorized_keys" < "$HOME"/.ssh/id_rsa.pub + sudo chmod 600 /root/.ssh/authorized_keys +} + +function _configure_setup_heketi { + # get Heketi and start service + wget "$HEKETI_V1_PACKAGE" + tar xvf "$(basename "$HEKETI_V1_PACKAGE")" + ( ./heketi/heketi -config "$GLUSTERFS_PLUGIN_DIR"/extras/heketi.json &>/dev/null & ) & + + # basic Heketi setup + $GLUSTERFS_PLUGIN_DIR/extras/heketisetup.py -s 1T -n 3 -v -D $(hostname) +} + +function _configure_manila_glusterfs_heketi { + _setup_rootssh + _configure_setup_heketi + + # Manila config + local share_driver=manila.share.drivers.glusterfs.GlusterfsShareDriver + local group_name=glusterheketi1 + + iniset $MANILA_CONF $group_name share_driver $share_driver + iniset $MANILA_CONF $group_name share_backend_name GLUSTERFSHEKETI + iniset $MANILA_CONF $group_name driver_handles_share_servers False + iniset $MANILA_CONF $group_name glusterfs_share_layout layout_heketi.GlusterfsHeketiLayout + iniset $MANILA_CONF $group_name glusterfs_heketi_url http://localhost:8080 + iniset $MANILA_CONF $group_name glusterfs_heketi_nodeadmin_username root + iniset $MANILA_CONF $group_name glusterfs_heketi_volume_replica 1 +} + # Configure GlusterFS as a backend for Manila function configure_manila_backend_glusterfs { - if [[ "${GLUSTERFS_MANILA_DRIVER_TYPE}" == "glusterfs" ]]; then + if [[ "${GLUSTERFS_MANILA_DRIVER_TYPE}" == "glusterfs-heketi" ]]; then + _configure_manila_glusterfs_heketi + elif [[ "${GLUSTERFS_MANILA_DRIVER_TYPE}" == "glusterfs" ]]; then _configure_manila_glusterfs_nfs else _configure_manila_glusterfs_native diff --git a/devstack/settings b/devstack/settings index 25ee9e7..1ed85fd 100644 --- a/devstack/settings +++ b/devstack/settings @@ -25,6 +25,9 @@ if [[ ${DISTRO} =~ rhel7 ]] && [[ ! -f /etc/yum.repos.d/glusterfs-epel.repo ]]; GLUSTERFS_CENTOS_REPO=${GLUSTERFS_CENTOS_REPO:-"http://download.gluster.org/pub/gluster/glusterfs/LATEST/CentOS/glusterfs-epel.repo"} fi +# Official Heketi 1.0.* binary +HEKETI_V1_PACKAGE="https://github.com/heketi/heketi/releases/download/1.0.2/heketi-1.0.2-release-1.0.0.linux.amd64.tar.gz" + TEMPEST_STORAGE_PROTOCOL=glusterfs ######### Glance Specific Configuration ######### diff --git a/extras/heketi.json b/extras/heketi.json new file mode 100644 index 0000000..dde9921 --- /dev/null +++ b/extras/heketi.json @@ -0,0 +1,35 @@ +{ + "_port_comment": "Heketi Server Port Number", + "port" : "8080", + + "_use_auth": "Enable JWT authorization. Please enable for deployment", + "use_auth" : false, + + "_jwt" : "Private keys for access", + "jwt" : { + "_admin" : "Admin has access to all APIs", + "admin" : { + "key" : "My Secret" + }, + "_user" : "User only has access to /volumes endpoint", + "user" : { + "key" : "My Secret" + } + }, + + "_glusterfs_comment": "GlusterFS Configuration", + "glusterfs" : { + + "_executor_comment": "Execute plugin. Possible choices: mock, ssh", + "executor" : "ssh", + + "_db_comment": "Database file name", + "db" : "heketi.db", + + "sshexec": { + "user": "root" + }, + "brick_min_size_gb": 1 + + } +} diff --git a/extras/heketisetup.py b/extras/heketisetup.py new file mode 100755 index 0000000..567ebb0 --- /dev/null +++ b/extras/heketisetup.py @@ -0,0 +1,288 @@ +#!/usr/bin/env python + +# Copyright (c) 2016 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import print_function +import datetime +import hashlib +import pipes +import re +import subprocess +import sys +import time + +try: + import jwt +except ImportError: + print("jwt module not found, Heketi JWT auth won't work.\n" + "You can install it with 'pip install PyJWT'.\n", file=sys.stderr) +import requests + + +class Objlog(object): + + def __init__(self, obj): + self.object = obj + self.__log__ = [] + + def __getattr__(self, att): + av = getattr(self.object, att) + + if not hasattr(av, '__call__'): + return av + + def alog(*a, **kw): + try: + ret = av(*a, **kw) + except Exception as x: + self.__log__.append(((att, a, kw), None, x)) + raise x + self.__log__.append(((att, a, kw), ret)) + return ret + + return alog + + def __reset__(self): + self.__log__ = [] + + +def objlog_get(ol): + return ol.__log__ + + +def reqlog(req, lower=0, upper=None): + ol = objlog_get(req) + if upper is None: + upper = len(ol) + for i in range(lower, upper): + e = ol[i] + print(i, e[0]) + print(e[1], e[1].headers, e[1].text) + + +class HeketiException(Exception): + pass + + +def jwt_token(method, path, key, issuer="admin", delta={'minutes': 5}): + method = method.upper() + path = re.sub("\A/+", "/", "/" + path) + + claims = {} + + # Issuer + claims['iss'] = issuer + + # Issued at time + claims['iat'] = datetime.datetime.utcnow() + + # Expiration time + claims['exp'] = (datetime.datetime.utcnow() + + datetime.timedelta(**delta)) + + # URI tampering protection + claims['qsh'] = hashlib.sha256(method + '&' + path).hexdigest() + + return jwt.encode(claims, key, algorithm='HS256') + + +class HeketiClient(object): + """A client class for the Heteki GlusterFS management service.""" + + def __init__(self, host, requests_like=requests, jwt_key=None): + host_stripped = re.sub("/+\Z", "", host) + self.__host__ = host_stripped + self.__requests__ = requests_like + self.__jwt_key__ = jwt_key + + def __getattr__(self, attr): + attr_value = getattr(self.__requests__, attr) + + if not hasattr(attr_value, '__call__'): + return attr_value + + def _attr_value_host_injected(*a, **kw): + if len(a) == 0: + return attr_value(*a, **kw) + else: + path = a[0] + path_stripped = re.sub("\A/+", "", path) + req_url = "/".join((self.__host__, path_stripped)) + if self.__jwt_key__: + token = jwt_token(attr, path, self.__jwt_key__) + kw['headers'] = {"Authorization": "bearer %s" % token} + report("Heketi request %(method)s to %(url)s" % { + 'method': attr.upper(), 'url': req_url}) + resp = attr_value(req_url, *a[1:], **kw) + report("Heketi response: %s" % repr(resp)) + return resp + + return _attr_value_host_injected + + def asyncop(self, *a, **kw): + method = kw.pop('method', 'post') + retry_interval = kw.pop('retry_interval', 1) + resp = getattr(self, method)(*a, **kw) + if resp.status_code != 202: + resp.raise_for_status() + raise HeketiException(( + 'Unexpected Heketi async %(method)s status %(status)d' + ) % {'method': method.upper(), 'status': resp.status_code}) + queue = resp.headers['location'] + while True: + resp = self.get(queue) + if resp.status_code == 204: + return resp + elif resp.status_code == 303: + return self.get(resp.headers['location']) + elif resp.status_code == 200: + if 'x-pending' not in resp.headers: + return resp + else: + resp.raise_for_status() + raise HeketiException(( + 'Unexpected Heketi async queue status %d' + ) % resp.status_code) + time.sleep(retry_interval) + + +class ShExec(object): + + def __init__(self, host, user=None, key=None, root=False): + self.host = host + self.root = root + self.args = [] + if host == "localhost": + self.args = ["sh", "-c"] + else: + self.args = ["ssh", host] + if user: + self.args.extend(["-l", user]) + if key: + self.args.extend(["-i", key]) + + def __call__(self, cmd): + if not self.root: + cmd = "sudo sh -c " + pipes.quote(cmd) + report("Running on %(host)s: %(cmd)s" % { + 'cmd': cmd, 'host': self.host}) + po = subprocess.Popen(self.args + [cmd], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = po.communicate() + report("Command output: %s" % out, cond=out) + if po.returncode: + raise RuntimeError("%(cmd)s has failed with %(exit)d" % { + 'cmd': ' '.join(self.args) + ' ' + cmd, + 'exit': po.returncode}) + return out, err + + +def setup(clusters, args, h): + cluster = args.cluster + if not cluster and clusters: + cluster = clusters[0] + if cluster not in clusters: + if re.match('[\da-f]{8}(-?[\da-f]{4}){3}-?[\da-f]{12}\Z', + cluster or '', re.I): + raise HeketiException( + "Cluster %s not found on Heketi server." % cluster) + cluster = None + if not cluster: + cluster = h.post("clusters").json()['id'] + report("Using cluster %s" % cluster) + + hostdevices = {} + for host in args.host: + hostdevices[host] = [] + shx = ShExec(host, user=args.user, key=args.key, root=args.root) + for i in range(args.devices): + dev, _ = shx("i=0; while [ -f /LOOP%(cluster)s-$i ]; do i=$(($i+1)); done && " + "truncate -s %(size)s /LOOP%(cluster)s-$i && " + "losetup -f --show /LOOP%(cluster)s-$i" % {'size': args.size, + 'cluster': cluster}) + hostdevices[host].append(dev.strip()) + + for host, devices in hostdevices.items(): + # add a node + node = h.asyncop("nodes", json={ + "zone": 1, + "hostnames": {"manage": [host], "storage": [host]}, + "cluster": cluster}).json() + for dev in devices: + # add a device + h.asyncop("devices", json={"node": node['id'], "name": dev}) + + +def teardown(cliusters, args, h): + if args.cluster not in clusters: + raise HeketiException( + "Cluster %s not found on Heketi server." % args.cluster) + + cluster = h.get("clusters/%s" % args.cluster).json() + for vol in cluster["volumes"]: + h.asyncop('volumes/%s' % vol, method='delete') + for nodeid in cluster["nodes"]: + node = h.get("nodes/%s" % nodeid).json() + for dev in node["devices"]: + h.asyncop('devices/%s' % dev['id'], method='delete') + h.asyncop('nodes/%s' % nodeid, method='delete') + h.delete("clusters/%s" % args.cluster) + +if __name__ == '__main__': + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("host", nargs='+') + parser.add_argument("-v", "--verbose", action='store_true') + parser.add_argument("-H", "--heketi", default="http://localhost:8080", + help="Heketi service URL") + parser.add_argument("-s", "--size", help="size of devices created") + parser.add_argument("-n", "--devices", + help="number of devices created per node", type=int) + parser.add_argument("-c", "--cluster", help="Heketi cluster to use") + parser.add_argument("-j", "--jwt", help="JWT key to use with Heketi auth") + parser.add_argument("-u", "--user", default="heketi", + help="user with which nodes are managed") + parser.add_argument("-k", "--key", help="SSH key used to log in remotely") + parser.add_argument("--root", action='store_true', + help="remote user has root privileges") + parser.add_argument("-A", "--action", choices=('setup', 'teardown'), + default='setup') + parser.add_argument("-D", "--debug", action='store_true') + args = parser.parse_args() + + if args.jwt: + jwt + if args.verbose: + def report(*a, **kw): + if not kw.pop('cond', True): + return + print(*a, **kw) + else: + report = lambda *a, **kw: None + + req = requests.session() + if args.debug: + req = Objlog(req) + heketi = HeketiClient(args.heketi, requests_like=req, jwt_key=args.jwt) + + try: + # get a cluster + clusters = heketi.get("clusters").json()['clusters'] + getattr(sys.modules[__name__], args.action)(clusters, args, heketi) + finally: + if args.debug: + reqlog(req) diff --git a/manila/post_test_hook.sh b/manila/post_test_hook.sh index d70dc89..601660c 100755 --- a/manila/post_test_hook.sh +++ b/manila/post_test_hook.sh @@ -35,7 +35,11 @@ if [[ "$JOB_NAME" =~ "glusterfs-native" ]]; then iniset $TEMPEST_CONFIG share enable_cert_rules_for_protocols glusterfs iniset $TEMPEST_CONFIG share capability_snapshot_support True else - local BACKEND_NAME="GLUSTERFS" + if [[ "$JOB_NAME" =~ "glusterfs-heketi" ]]; then + local BACKEND_NAME="GLUSTERFSHEKETI" + else + local BACKEND_NAME="GLUSTERFS" + fi iniset $TEMPEST_CONFIG share enable_protocols nfs iniset $TEMPEST_CONFIG share enable_ip_rules_for_protocols nfs iniset $TEMPEST_CONFIG share storage_protocol NFS