From 5dca3b0b9362721b7fdd80788368caf02ab6db0c Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 4 Oct 2012 14:24:12 +0100 Subject: [PATCH] Updated README verbosity, added checks to harden ceph admin-daemon usage in ceph utils --- README | 81 +++++++++++++++++++++++++++++++++++++++++++-------- TODO | 8 +++++ config.yaml | 2 +- copyright | 4 +-- hooks/ceph.py | 32 ++++++++++++-------- revision | 2 +- 6 files changed, 101 insertions(+), 28 deletions(-) diff --git a/README b/README index 0b56260b..2d1159dd 100644 --- a/README +++ b/README @@ -1,18 +1,75 @@ +Overview +======== + +Ceph is a distributed storage and network file system designed to provide +excellent performance, reliability, and scalability. + This charm deploys a Ceph cluster. -It uses the new-style Ceph deployment as reverse-engineered from the -Chef cookbook at https://github.com/ceph/ceph-cookbooks +Usage +===== + +The ceph charm has two pieces of mandatory configuration for which no defaults +are provided: + + fsid: + uuid specific to a ceph cluster used to ensure that different + clusters don't get mixed up - use `uuid` to generate one. + + monitor-secret: + a ceph generated key used by the daemons that manage to cluster + to control security. You can use the ceph-authtool command to + generate one: + + ceph-authtool /dev/stdout --name=mon. --gen-key + +These two pieces of configuration must NOT be changed post bootstrap; attempting +todo this will cause a reconfiguration error and new service units will not join +the existing ceph cluster. + +The charm also supports specification of the storage devices to use in the ceph +cluster. + + osd-devices: + A list of devices that the charm will attempt to detect, initialise and + activate as ceph storage. + + This this can be a superset of the actual storage devices presented to + each service unit and can be changed post ceph bootstrap using `juju set`. + +At a minimum you must provide a juju config file during initial deployment +with the fsid and monitor-secret options: + + ceph: + fsid: ecbb8960-0e21-11e2-b495-83a88f44db01 + monitor-secret: AQD1P2xQiKglDhAA4NGUF5j38Mhq56qwz+45wg== + osd-devices: /dev/vdb /dev/vdc /dev/vdd /dev/vde + +Specifying the osd-devices to use is also a good idea. + +By default the ceph cluster will not bootstrap until 3 service units have been +deployed and started; this is to ensure that a quorum is achieved prior to adding +storage devices. + +Bootnotes +========= + +This charm uses the new-style Ceph deployment as reverse-engineered from the Chef +cookbook at https://github.com/ceph/ceph-cookbooks. + +This charm is currently deliberately inflexible and potentially destructive. It +is designed to deploy on exactly three machines. -This charm is currently deliberately inflexible and potentially -destructive. It is designed to deploy on exactly three machines. Each machine will run mon and osd. -The osds use so-called "OSD hotplugging". ceph-disk-prepare is used -to create the filesystems with a special GPT partition type. udev is -set up to mount such filesystems and start the osd daemons as their -storage becomes visible to the system (or after "udevadm trigger"). +The osds use so-called "OSD hotplugging". ceph-disk-prepare is used to create the +filesystems with a special GPT partition type. udev is set up to mount such +filesystems and start the osd daemons as their storage becomes visible to the +system (or after "udevadm trigger"). -The Chef cookbook above performs some extra steps to generate an OSD -bootstrapping key and propagate it to the other nodes in the cluster. -Since all our OSDs run on nodes that also run mon, we don't need this -and did not implement it. +The Chef cookbook above performs some extra steps to generate an OSD bootstrapping +key and propagate it to the other nodes in the cluster. Since all our OSDs run on +nodes that also run mon, we don't need this and did not implement it. + +The charm does not currently implement cephx and its explicitly turned off in the +configuration generated for ceph. diff --git a/TODO b/TODO index c4247485..8fcf6066 100644 --- a/TODO +++ b/TODO @@ -2,7 +2,15 @@ * fix tunables (http://tracker.newdream.net/issues/2210) * more than 192 PGs + * fixup data placement in crush to be host not osd driven == Major == * deploy more than 3 OSD hosts + +== Public Charm == + + * cephx support + * rel: remote OSD services (+bootstrap.osd keys for cephx) + * rel: remote MON clients (+client keys for cephx) + * rel: RADOS gateway (+client key for cephx) diff --git a/config.yaml b/config.yaml index a59f67e6..b8314087 100644 --- a/config.yaml +++ b/config.yaml @@ -9,7 +9,7 @@ options: monitor-secret: type: string description: | - This value will become the "mon." key. To generate a suitable value use: + This value will become the mon. key. To generate a suitable value use: . ceph-authtool /dev/stdout --name=mon. --gen-key . diff --git a/copyright b/copyright index 4e1085af..bdfae0e0 100644 --- a/copyright +++ b/copyright @@ -11,5 +11,5 @@ Copyright: 2004-2010 by Sage Weil License: LGPL-2.1 License: LGPL-2.1 - On Debian GNU/Linux system you can find the complete text of the - LGPL-2.1 license in '/usr/share/common-licenses/LGPL-2.1' + On Debian GNU/Linux system you can find the complete text of the + LGPL-2.1 license in '/usr/share/common-licenses/LGPL-2.1' diff --git a/hooks/ceph.py b/hooks/ceph.py index 83b113cf..f542ed9a 100644 --- a/hooks/ceph.py +++ b/hooks/ceph.py @@ -11,25 +11,31 @@ import json import subprocess import time import utils +import os QUORUM = ['leader', 'peon'] def is_quorum(): + asok = "/var/run/ceph/ceph-mon.{}.asok".format(utils.get_unit_hostname()) cmd = [ "ceph", "--admin-daemon", - "/var/run/ceph/ceph-mon.{}.asok".format(utils.get_unit_hostname()), + asok, "mon_status" ] - - try: - result = json.loads(subprocess.check_output(cmd)) - except subprocess.CalledProcessError: - return False - - if result['state'] in QUORUM: - return True + if os.path.exists(asok): + try: + result = json.loads(subprocess.check_output(cmd)) + except subprocess.CalledProcessError: + return False + except ValueError: + # Non JSON response from mon_status + return False + if result['state'] in QUORUM: + return True + else: + return False else: return False @@ -40,12 +46,14 @@ def wait_for_quorum(): def add_bootstrap_hint(peer): + asok = "/var/run/ceph/ceph-mon.{}.asok".format(utils.get_unit_hostname()) cmd = [ "ceph", "--admin-daemon", - "/var/run/ceph/ceph-mon.{}.asok".format(utils.get_unit_hostname()), + asok, "add_bootstrap_peer_hint", peer ] - # Ignore any errors for this call - subprocess.call(cmd) + if os.path.exists(asok): + # Ignore any errors for this call + subprocess.call(cmd) diff --git a/revision b/revision index fb1e7bc8..c3f407c0 100644 --- a/revision +++ b/revision @@ -1 +1 @@ -54 +55