diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 7f7b5be..0000000 --- a/.coveragerc +++ /dev/null @@ -1,7 +0,0 @@ -[report] -# Regexes for lines to exclude from consideration -exclude_lines = - if __name__ == .__main__.: -include= - hooks/hooks.py - hooks/ceph*.py diff --git a/.project b/.project deleted file mode 100644 index be5d420..0000000 --- a/.project +++ /dev/null @@ -1,17 +0,0 @@ - - - ceph - - - - - - org.python.pydev.PyDevBuilder - - - - - - org.python.pydev.pythonNature - - diff --git a/.pydevproject b/.pydevproject deleted file mode 100644 index 998e0aa..0000000 --- a/.pydevproject +++ /dev/null @@ -1,8 +0,0 @@ - - -python 2.7 -Default - -/ceph/hooks - - diff --git a/.testr.conf b/.testr.conf deleted file mode 100644 index 801646b..0000000 --- a/.testr.conf +++ /dev/null @@ -1,8 +0,0 @@ -[DEFAULT] -test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \ - OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \ - OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \ - ${PYTHON:-python} -m subunit.run discover -t ./ ./unit_tests $LISTOPT $IDOPTION - -test_id_option=--load-list $IDFILE -test_list_option=--list diff --git a/LICENSE b/LICENSE deleted file mode 100644 index d645695..0000000 --- a/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/Makefile b/Makefile deleted file mode 100644 index 006276b..0000000 --- a/Makefile +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/make -PYTHON := /usr/bin/env python - -lint: - @tox -e pep8 - -test: - @echo Starting unit tests... - @tox -e py27 - -functional_test: - @echo Starting Amulet tests... - @tox -e func27 - -bin/charm_helpers_sync.py: - @mkdir -p bin - @curl -o bin/charm_helpers_sync.py https://raw.githubusercontent.com/juju/charm-helpers/master/tools/charm_helpers_sync/charm_helpers_sync.py - - -bin/git_sync.py: - @mkdir -p bin - @wget -O bin/git_sync.py https://raw.githubusercontent.com/CanonicalLtd/git-sync/master/git_sync.py - -ch-sync: bin/charm_helpers_sync.py - $(PYTHON) bin/charm_helpers_sync.py -c charm-helpers-hooks.yaml - $(PYTHON) bin/charm_helpers_sync.py -c charm-helpers-tests.yaml - -ceph-sync: bin/git_sync.py - $(PYTHON) bin/git_sync.py -d lib -s https://github.com/openstack/charms.ceph.git - -sync: ch-sync - -publish: lint test - bzr push lp:charms/ceph - bzr push lp:charms/trusty/ceph diff --git a/README b/README new file mode 100644 index 0000000..b72ee4f --- /dev/null +++ b/README @@ -0,0 +1,6 @@ +This project is no longer maintained. + +The contents of this repository are still available in the Git +source code management system. To see the contents of this +repository before it reached its end of life, please check out the +previous commit with "git checkout HEAD^1". diff --git a/README.md b/README.md deleted file mode 100644 index 474f3e8..0000000 --- a/README.md +++ /dev/null @@ -1,143 +0,0 @@ -# Overview - ---- -**NOTE** - -This charm is deprecated and will not receive updates past February 2018. - -Existing users should refer to [Appendix A](https://docs.openstack.org/project-deploy-guide/charm-deployment-guide/latest/app-ceph-migration.html). -of the Charm Deployment Guide for details of how to migration existing -deployments to the preferred ceph-mon and ceph-osd charms. - ---- - -Ceph is a distributed storage and network file system designed to provide -excellent performance, reliability, and scalability. - -This charm deploys a Ceph cluster. - -# Usage - -The ceph charm has two pieces of mandatory configuration for which no defaults -are provided. You _must_ set these configuration options before deployment or the charm will not work: - - fsid: - uuid specific to a ceph cluster used to ensure that different - clusters don't get mixed up - use `uuid` to generate one. - - monitor-secret: - a ceph generated key used by the daemons that manage to cluster - to control security. You can use the ceph-authtool command to - generate one: - - ceph-authtool /dev/stdout --name=mon. --gen-key - -These two pieces of configuration must NOT be changed post bootstrap; attempting -to do this will cause a reconfiguration error and new service units will not join -the existing ceph cluster. - -The charm also supports the specification of storage devices to be used in the -ceph cluster. - - osd-devices: - A list of devices that the charm will attempt to detect, initialise and - activate as ceph storage. - - This can be a superset of the actual storage devices presented to each - service unit and can be changed post ceph bootstrap using `juju set`. - - The full path of each device must be provided, e.g. /dev/vdb. - - For Ceph >= 0.56.6 (Raring or the Grizzly Cloud Archive) use of - directories instead of devices is also supported. - -At a minimum you must provide a juju config file during initial deployment -with the fsid and monitor-secret options (contents of cepy.yaml below): - - ceph: - fsid: ecbb8960-0e21-11e2-b495-83a88f44db01 - monitor-secret: AQD1P2xQiKglDhAA4NGUF5j38Mhq56qwz+45wg== - osd-devices: /dev/vdb /dev/vdc /dev/vdd /dev/vde - -Specifying the osd-devices to use is also a good idea. - -Boot things up by using: - - juju deploy -n 3 --config ceph.yaml ceph - -By default the ceph cluster will not bootstrap until 3 service units have been -deployed and started; this is to ensure that a quorum is achieved prior to adding -storage devices. - -## Actions - -This charm supports pausing and resuming ceph's health functions on a cluster, for example when doing maintenance on a machine. to pause or resume, call: - -`juju action do --unit ceph/0 pause-health` or `juju action do --unit ceph/0 resume-health` - -## Scale Out Usage - -You can use the Ceph OSD and Ceph Radosgw charms: - -- [Ceph OSD](https://jujucharms.com/ceph-osd) -- [Ceph Rados Gateway](https://jujucharms.com/ceph-radosgw) - -## Network Space support - -This charm supports the use of Juju Network Spaces, allowing the charm to be bound to network space configurations managed directly by Juju. This is only supported with Juju 2.0 and above. - -Network traffic can be bound to specific network spaces using the public (front-side) and cluster (back-side) bindings: - - juju deploy ceph --bind "public=data-space cluster=cluster-space" - -alternatively these can also be provided as part of a Juju native bundle configuration: - - ceph: - charm: cs:xenial/ceph - num_units: 1 - bindings: - public: data-space - cluster: cluster-space - -Please refer to the [Ceph Network Reference](http://docs.ceph.com/docs/master/rados/configuration/network-config-ref) for details on how using these options effects network traffic within a Ceph deployment. - -**NOTE:** Spaces must be configured in the underlying provider prior to attempting to use them. - -**NOTE**: Existing deployments using ceph-*-network configuration options will continue to function; these options are preferred over any network space binding provided if set. - -# Contact Information - -## Authors - -- Paul Collins , -- James Page - -Report bugs on [Launchpad](http://bugs.launchpad.net/charms/+source/ceph/+filebug) - -## Ceph - -- [Ceph website](http://ceph.com) -- [Ceph mailing lists](http://ceph.com/resources/mailing-list-irc/) -- [Ceph bug tracker](http://tracker.ceph.com/projects/ceph) - -# Technical Footnotes - -This charm uses the new-style Ceph deployment as reverse-engineered from the -Chef cookbook at https://github.com/ceph/ceph-cookbooks, although we selected -a different strategy to form the monitor cluster. Since we don't know the -names *or* addresses of the machines in advance, we use the _relation-joined_ -hook to wait for all three nodes to come up, and then write their addresses -to ceph.conf in the "mon host" parameter. After we initialize the monitor -cluster a quorum forms quickly, and OSD bringup proceeds. - -The osds use so-called "OSD hotplugging". **ceph-disk prepare** is used to -create the filesystems with a special GPT partition type. *udev* is set up -to mount such filesystems and start the osd daemons as their storage becomes -visible to the system (or after `udevadm trigger`). - -The Chef cookbook mentioned above performs some extra steps to generate an OSD -bootstrapping key and propagate it to the other nodes in the cluster. Since -all OSDs run on nodes that also run mon, we don't need this and did not -implement it. - -See [the documentation](http://ceph.com/docs/master/dev/mon-bootstrap/) for more information on Ceph monitor cluster deployment strategies and pitfalls. diff --git a/TODO b/TODO deleted file mode 100644 index 22e0889..0000000 --- a/TODO +++ /dev/null @@ -1,6 +0,0 @@ -Ceph Charm -========== - - * fix tunables (http://tracker.newdream.net/issues/2210) - * more than 192 PGs - * fixup data placement in crush to be host not osd driven diff --git a/actions.yaml b/actions.yaml deleted file mode 100644 index d26a41e..0000000 --- a/actions.yaml +++ /dev/null @@ -1,217 +0,0 @@ -pause-health: - description: Pause ceph health operations across the entire ceph cluster -resume-health: - description: Resume ceph health operations across the entire ceph cluster -pause: - description: | - CAUTION - Set the local osd units in the charm to 'out' but does not stop - the osds. Unless the osd cluster is set to noout (see below), this removes - them from the ceph cluster and forces ceph to migrate the PGs to other OSDs - in the cluster. See the following. - - http://docs.ceph.com/docs/master/rados/operations/add-or-rm-osds/#removing-the-osd - "Do not let your cluster reach its full ratio when removing an OSD. - Removing OSDs could cause the cluster to reach or exceed its full ratio." - Also note that for small clusters you may encounter the corner case where - some PGs remain stuck in the active+remapped state. Refer to the above link - on how to resolve this. - - pause-health unit can be used before pausing the ceph units to stop the - cluster rebalancing the data off this unit. pause-health sets 'noout' on - the cluster such that it will not try to rebalance the data accross the - remaining units. - - It is up to the user of the charm to determine whether pause-health should - be used as it depends on whether the osd is being paused for maintenance or - to remove it from the cluster completely. -resume: - description: | - Set the local osd units in the charm to 'in'. Note that the pause option - does NOT stop the osd processes. -create-pool: - description: Creates a pool - params: - name: - type: string - description: The name of the pool - profile-name: - type: string - description: The crush profile to use for this pool. The ruleset must exist first. - pool-type: - type: string - default: "replicated" - enum: [replicated, erasure] - description: | - The pool type which may either be replicated to recover from lost OSDs by keeping multiple copies of the - objects or erasure to get a kind of generalized RAID5 capability. - replicas: - type: integer - default: 3 - description: | - For the replicated pool this is the number of replicas to store of each object. - erasure-profile-name: - type: string - default: default - description: | - The name of the erasure coding profile to use for this pool. Note this profile must exist - before calling create-pool - required: [name] - additionalProperties: false -create-erasure-profile: - description: Create a new erasure code profile to use on a pool. - params: - name: - type: string - description: The name of the profile - failure-domain: - type: string - default: host - enum: [chassis, datacenter, host, osd, pdu, pod, rack, region, room, root, row] - description: | - The failure-domain=host will create a CRUSH ruleset that ensures no two chunks are stored in the same host. - plugin: - type: string - default: "jerasure" - enum: [jerasure, isa, lrc, shec] - description: | - The erasure plugin to use for this profile. - See http://docs.ceph.com/docs/master/rados/operations/erasure-code-profile/ for more details - data-chunks: - type: integer - default: 3 - description: | - The number of data chunks, i.e. the number of chunks in which the original object is divided. For instance - if K = 2 a 10KB object will be divided into K objects of 5KB each. - coding-chunks: - type: integer - default: 2 - description: | - The number of coding chunks, i.e. the number of additional chunks computed by the encoding functions. - If there are 2 coding chunks, it means 2 OSDs can be out without losing data. - locality-chunks: - type: integer - description: | - Group the coding and data chunks into sets of size locality. For instance, for k=4 and m=2, when locality=3 - two groups of three are created. Each set can be recovered without reading chunks from another set. - durability-estimator: - type: integer - description: | - The number of parity chunks each of which includes each data chunk in its calculation range. The number is used - as a durability estimator. For instance, if c=2, 2 OSDs can be down without losing data. - required: [name, data-chunks, coding-chunks] - additionalProperties: false -get-erasure-profile: - description: Display an erasure code profile. - params: - name: - type: string - description: The name of the profile - required: [name] - additionalProperties: false -delete-erasure-profile: - description: Deletes an erasure code profile. - params: - name: - type: string - description: The name of the profile - required: [name] - additionalProperties: false -list-erasure-profiles: - description: List the names of all erasure code profiles - additionalProperties: false -list-pools: - description: List your cluster’s pools - additionalProperties: false -set-pool-max-bytes: - description: Set pool quotas for the maximum number of bytes. - params: - max: - type: integer - description: The name of the pool - pool-name: - type: string - description: The name of the pool - required: [pool-name, max] - additionalProperties: false -delete-pool: - description: Deletes the named pool - params: - pool-name: - type: string - description: The name of the pool - required: [pool-name] - additionalProperties: false -rename-pool: - description: Rename a pool - params: - pool-name: - type: string - description: The name of the pool - new-name: - type: string - description: The new name of the pool - required: [pool-name, new-name] - additionalProperties: false -pool-statistics: - description: Show a pool’s utilization statistics - additionalProperties: false -snapshot-pool: - description: Snapshot a pool - params: - pool-name: - type: string - description: The name of the pool - snapshot-name: - type: string - description: The name of the snapshot - required: [snapshot-name, pool-name] - additionalProperties: false -remove-pool-snapshot: - description: Remove a pool snapshot - params: - pool-name: - type: string - description: The name of the pool - snapshot-name: - type: string - description: The name of the snapshot - required: [snapshot-name, pool-name] - additionalProperties: false -pool-set: - description: Set a value for the pool - params: - pool-name: - type: string - description: The pool to set this variable on. - key: - type: string - description: Any valid Ceph key from http://docs.ceph.com/docs/master/rados/operations/pools/#set-pool-values - value: - type: string - description: The value to set - required: [key, value, pool-name] - additionalProperties: false -pool-get: - description: Get a value for the pool - params: - pool-name: - type: string - description: The pool to get this variable from. - key: - type: string - description: Any valid Ceph key from http://docs.ceph.com/docs/master/rados/operations/pools/#get-pool-values - required: [key, pool-name] - additionalProperties: false -list-disks: - description: List the unmounted disk on the specified unit -add-disk: - description: Add disk(s) to Ceph - params: - osd-devices: - type: string - description: The devices to format and set up as osd volumes. - bucket: - type: string - description: The name of the bucket in Ceph to add these devices into - required: - - osd-devices diff --git a/actions/__init__.py b/actions/__init__.py deleted file mode 100644 index b7fe4e1..0000000 --- a/actions/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -sys.path.append('hooks') diff --git a/actions/add-disk b/actions/add-disk deleted file mode 120000 index 4379d79..0000000 --- a/actions/add-disk +++ /dev/null @@ -1 +0,0 @@ -add_disk.py \ No newline at end of file diff --git a/actions/add_disk.py b/actions/add_disk.py deleted file mode 100755 index b545e18..0000000 --- a/actions/add_disk.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/python -# -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import psutil -import sys - -sys.path.append('lib') -sys.path.append('hooks') - -from charmhelpers.core.hookenv import ( - config, - action_get, -) - -from charmhelpers.contrib.storage.linux.ceph import ( - CephBrokerRq, - send_request_if_needed, -) - -import ceph.utils as ceph - -from ceph_hooks import ( - get_journal_devices, -) - - -def add_device(request, device_path, bucket=None): - ceph.osdize(dev, config('osd-format'), - get_journal_devices(), config('osd-reformat'), - config('ignore-device-errors'), - config('osd-encrypt'), - config('bluestore')) - # Make it fast! - if config('autotune'): - ceph.tune_dev(dev) - mounts = filter(lambda disk: device_path - in disk.device, psutil.disk_partitions()) - if mounts: - osd = mounts[0] - osd_id = osd.mountpoint.split('/')[-1].split('-')[-1] - request.ops.append({ - 'op': 'move-osd-to-bucket', - 'osd': "osd.{}".format(osd_id), - 'bucket': bucket}) - return request - - -def get_devices(): - devices = [] - for path in action_get('osd-devices').split(' '): - path = path.strip() - if os.path.isabs(path): - devices.append(path) - - return devices - - -if __name__ == "__main__": - request = CephBrokerRq() - for dev in get_devices(): - request = add_device(request=request, - device_path=dev, - bucket=action_get("bucket")) - send_request_if_needed(request, relation='mon') diff --git a/actions/ceph_ops.py b/actions/ceph_ops.py deleted file mode 100755 index c4df90f..0000000 --- a/actions/ceph_ops.py +++ /dev/null @@ -1,116 +0,0 @@ -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from subprocess import CalledProcessError, check_output -import sys - -sys.path.append('hooks') - -import rados -from charmhelpers.core.hookenv import log, action_get, action_fail -from charmhelpers.contrib.storage.linux.ceph import pool_set, \ - set_pool_quota, snapshot_pool, remove_pool_snapshot - - -# Connect to Ceph via Librados and return a connection -def connect(): - try: - cluster = rados.Rados(conffile='/etc/ceph/ceph.conf') - cluster.connect() - return cluster - except (rados.IOError, - rados.ObjectNotFound, - rados.NoData, - rados.NoSpace, - rados.PermissionError) as rados_error: - log("librados failed with error: {}".format(str(rados_error))) - - -def create_crush_rule(): - # Shell out - pass - - -def list_pools(): - try: - cluster = connect() - pool_list = cluster.list_pools() - cluster.shutdown() - return pool_list - except (rados.IOError, - rados.ObjectNotFound, - rados.NoData, - rados.NoSpace, - rados.PermissionError) as e: - action_fail(e.message) - - -def pool_get(): - key = action_get("key") - pool_name = action_get("pool_name") - try: - value = check_output(['ceph', 'osd', 'pool', 'get', pool_name, key]) - return value - except CalledProcessError as e: - action_fail(e.message) - - -def set_pool(): - key = action_get("key") - value = action_get("value") - pool_name = action_get("pool_name") - pool_set(service='ceph', pool_name=pool_name, key=key, value=value) - - -def pool_stats(): - try: - pool_name = action_get("pool-name") - cluster = connect() - ioctx = cluster.open_ioctx(pool_name) - stats = ioctx.get_stats() - ioctx.close() - cluster.shutdown() - return stats - except (rados.Error, - rados.IOError, - rados.ObjectNotFound, - rados.NoData, - rados.NoSpace, - rados.PermissionError) as e: - action_fail(e.message) - - -def delete_pool_snapshot(): - pool_name = action_get("pool-name") - snapshot_name = action_get("snapshot-name") - remove_pool_snapshot(service='ceph', - pool_name=pool_name, - snapshot_name=snapshot_name) - - -# Note only one or the other can be set -def set_pool_max_bytes(): - pool_name = action_get("pool-name") - max_bytes = action_get("max") - set_pool_quota(service='ceph', - pool_name=pool_name, - max_bytes=max_bytes) - - -def snapshot_ceph_pool(): - pool_name = action_get("pool-name") - snapshot_name = action_get("snapshot-name") - snapshot_pool(service='ceph', - pool_name=pool_name, - snapshot_name=snapshot_name) diff --git a/actions/create-erasure-profile b/actions/create-erasure-profile deleted file mode 100755 index 2b00b58..0000000 --- a/actions/create-erasure-profile +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/python -from subprocess import CalledProcessError -import sys - -sys.path.append('hooks') - -from charmhelpers.contrib.storage.linux.ceph import create_erasure_profile -from charmhelpers.core.hookenv import action_get, log, action_fail - - -def make_erasure_profile(): - name = action_get("name") - plugin = action_get("plugin") - failure_domain = action_get("failure-domain") - - # jerasure requires k+m - # isa requires k+m - # local requires k+m+l - # shec requires k+m+c - - if plugin == "jerasure": - k = action_get("data-chunks") - m = action_get("coding-chunks") - try: - create_erasure_profile(service='admin', - erasure_plugin_name=plugin, - profile_name=name, - data_chunks=k, - coding_chunks=m, - failure_domain=failure_domain) - except CalledProcessError as e: - log(e) - action_fail("Create erasure profile failed with " - "message: {}".format(e.message)) - elif plugin == "isa": - k = action_get("data-chunks") - m = action_get("coding-chunks") - try: - create_erasure_profile(service='admin', - erasure_plugin_name=plugin, - profile_name=name, - data_chunks=k, - coding_chunks=m, - failure_domain=failure_domain) - except CalledProcessError as e: - log(e) - action_fail("Create erasure profile failed with " - "message: {}".format(e.message)) - elif plugin == "local": - k = action_get("data-chunks") - m = action_get("coding-chunks") - l = action_get("locality-chunks") - try: - create_erasure_profile(service='admin', - erasure_plugin_name=plugin, - profile_name=name, - data_chunks=k, - coding_chunks=m, - locality=l, - failure_domain=failure_domain) - except CalledProcessError as e: - log(e) - action_fail("Create erasure profile failed with " - "message: {}".format(e.message)) - elif plugin == "shec": - k = action_get("data-chunks") - m = action_get("coding-chunks") - c = action_get("durability-estimator") - try: - create_erasure_profile(service='admin', - erasure_plugin_name=plugin, - profile_name=name, - data_chunks=k, - coding_chunks=m, - durability_estimator=c, - failure_domain=failure_domain) - except CalledProcessError as e: - log(e) - action_fail("Create erasure profile failed with " - "message: {}".format(e.message)) - else: - # Unknown erasure plugin - action_fail("Unknown erasure-plugin type of {}. " - "Only jerasure, isa, local or shec is " - "allowed".format(plugin)) - - -if __name__ == '__main__': - make_erasure_profile() diff --git a/actions/create-pool b/actions/create-pool deleted file mode 100755 index 4d1d214..0000000 --- a/actions/create-pool +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/python -import sys - -sys.path.append('hooks') -from subprocess import CalledProcessError -from charmhelpers.core.hookenv import action_get, log, action_fail -from charmhelpers.contrib.storage.linux.ceph import ErasurePool, ReplicatedPool - - -def create_pool(): - pool_name = action_get("name") - pool_type = action_get("pool-type") - try: - if pool_type == "replicated": - replicas = action_get("replicas") - replicated_pool = ReplicatedPool(name=pool_name, - service='admin', - replicas=replicas) - replicated_pool.create() - - elif pool_type == "erasure": - crush_profile_name = action_get("erasure-profile-name") - erasure_pool = ErasurePool(name=pool_name, - erasure_code_profile=crush_profile_name, - service='admin') - erasure_pool.create() - else: - log("Unknown pool type of {}. Only erasure or replicated is " - "allowed".format(pool_type)) - action_fail("Unknown pool type of {}. Only erasure or replicated " - "is allowed".format(pool_type)) - except CalledProcessError as e: - action_fail("Pool creation failed because of a failed process. " - "Ret Code: {} Message: {}".format(e.returncode, e.message)) - - -if __name__ == '__main__': - create_pool() diff --git a/actions/delete-erasure-profile b/actions/delete-erasure-profile deleted file mode 100755 index 075c410..0000000 --- a/actions/delete-erasure-profile +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/python -from subprocess import CalledProcessError - -__author__ = 'chris' -import sys - -sys.path.append('hooks') - -from charmhelpers.contrib.storage.linux.ceph import remove_erasure_profile -from charmhelpers.core.hookenv import action_get, log, action_fail - - -def delete_erasure_profile(): - name = action_get("name") - - try: - remove_erasure_profile(service='admin', profile_name=name) - except CalledProcessError as e: - action_fail("Remove erasure profile failed with error: {}".format( - e.message)) - - -if __name__ == '__main__': - delete_erasure_profile() diff --git a/actions/delete-pool b/actions/delete-pool deleted file mode 100755 index 3d65507..0000000 --- a/actions/delete-pool +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/python -import sys - -sys.path.append('hooks') - -import rados -from ceph_ops import connect -from charmhelpers.core.hookenv import action_get, log, action_fail - - -def remove_pool(): - try: - pool_name = action_get("name") - cluster = connect() - log("Deleting pool: {}".format(pool_name)) - cluster.delete_pool(str(pool_name)) # Convert from unicode - cluster.shutdown() - except (rados.IOError, - rados.ObjectNotFound, - rados.NoData, - rados.NoSpace, - rados.PermissionError) as e: - log(e) - action_fail(e) - - -if __name__ == '__main__': - remove_pool() diff --git a/actions/get-erasure-profile b/actions/get-erasure-profile deleted file mode 100755 index 29ece59..0000000 --- a/actions/get-erasure-profile +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/python -__author__ = 'chris' -import sys - -sys.path.append('hooks') - -from charmhelpers.contrib.storage.linux.ceph import get_erasure_profile -from charmhelpers.core.hookenv import action_get, action_set - - -def make_erasure_profile(): - name = action_get("name") - out = get_erasure_profile(service='admin', name=name) - action_set({'message': out}) - - -if __name__ == '__main__': - make_erasure_profile() diff --git a/actions/list-disks b/actions/list-disks deleted file mode 120000 index ebe3b65..0000000 --- a/actions/list-disks +++ /dev/null @@ -1 +0,0 @@ -list_disks.py \ No newline at end of file diff --git a/actions/list-erasure-profiles b/actions/list-erasure-profiles deleted file mode 100755 index cf6dfa0..0000000 --- a/actions/list-erasure-profiles +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/python -__author__ = 'chris' -import sys -from subprocess import check_output, CalledProcessError - -sys.path.append('hooks') - -from charmhelpers.core.hookenv import action_get, log, action_set, action_fail - -if __name__ == '__main__': - name = action_get("name") - try: - out = check_output(['ceph', - '--id', 'admin', - 'osd', - 'erasure-code-profile', - 'ls']).decode('UTF-8') - action_set({'message': out}) - except CalledProcessError as e: - log(e) - action_fail("Listing erasure profiles failed with error: {}".format( - e.message)) diff --git a/actions/list-pools b/actions/list-pools deleted file mode 100755 index 102667c..0000000 --- a/actions/list-pools +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/python -__author__ = 'chris' -import sys -from subprocess import check_output, CalledProcessError - -sys.path.append('hooks') - -from charmhelpers.core.hookenv import log, action_set, action_fail - -if __name__ == '__main__': - try: - out = check_output(['ceph', '--id', 'admin', - 'osd', 'lspools']).decode('UTF-8') - action_set({'message': out}) - except CalledProcessError as e: - log(e) - action_fail("List pools failed with error: {}".format(e.message)) diff --git a/actions/list_disks.py b/actions/list_disks.py deleted file mode 100755 index e5c0c0f..0000000 --- a/actions/list_disks.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/python -# -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -List unmounted devices. - -This script will get all block devices known by udev and check if they -are mounted so that we can give unmounted devices to the administrator. -""" - -import pyudev -import sys - - -sys.path.append('hooks/') - -from charmhelpers.contrib.storage.linux.utils import is_device_mounted -from charmhelpers.core.hookenv import log, action_set - - -if __name__ == '__main__': - disks = [] - context = pyudev.Context() - for device in context.list_devices(DEVTYPE='disk'): - if device['SUBSYSTEM'] == 'block': - matched = False - for block_type in [u'dm', u'loop', u'ram', u'nbd']: - if block_type in device.device_node: - matched = True - if matched: - continue - disks.append(device.device_node) - log("Found disks: {}".format(disks)) - unmounted_disks = [disk for disk in disks if not is_device_mounted(disk)] - - action_set({ - 'disks': unmounted_disks}) diff --git a/actions/pause b/actions/pause deleted file mode 120000 index bd4c0e0..0000000 --- a/actions/pause +++ /dev/null @@ -1 +0,0 @@ -pause_resume.py \ No newline at end of file diff --git a/actions/pause-health b/actions/pause-health deleted file mode 100755 index 207c4f6..0000000 --- a/actions/pause-health +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -set -eux - -ceph osd set nodown -ceph osd set noout \ No newline at end of file diff --git a/actions/pause_resume.py b/actions/pause_resume.py deleted file mode 100755 index 2d9023c..0000000 --- a/actions/pause_resume.py +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/python -# -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# pause/resume actions file. - -import os -import sys -from subprocess import check_call - -sys.path.append('hooks') -sys.path.append('lib') - -from charmhelpers.core.hookenv import ( - action_fail, -) - -from ceph.utils import get_local_osd_ids -from ceph_hooks import assess_status - -from utils import ( - set_unit_paused, - clear_unit_paused, -) - - -def pause(args): - """Pause the ceph-osd units on the local machine only. - - Optionally uses the 'osd-number' from juju action param to only pause a - specific osd. If all the osds are not stopped then the paused status is - not set. - - @raises CalledProcessError if the ceph commands fails. - @raises OSError if it can't get the local osd ids. - """ - for local_id in get_local_osd_ids(): - cmd = [ - 'ceph', - '--id', 'osd-upgrade', - 'osd', 'out', str(local_id)] - check_call(cmd) - set_unit_paused() - assess_status() - - -def resume(args): - """Resume the ceph-osd units on this local machine only - - @raises subprocess.CalledProcessError should the osd units fails to resume. - @raises OSError if the unit can't get the local osd ids - """ - for local_id in get_local_osd_ids(): - cmd = [ - 'ceph', - '--id', 'osd-upgrade', - 'osd', 'in', str(local_id)] - check_call(cmd) - clear_unit_paused() - assess_status() - - -# A dictionary of all the defined actions to callables (which take -# parsed arguments). -ACTIONS = {"pause": pause, "resume": resume} - - -def main(args): - action_name = os.path.basename(args[0]) - try: - action = ACTIONS[action_name] - except KeyError: - s = "Action {} undefined".format(action_name) - action_fail(s) - return s - else: - try: - action(args) - except Exception as e: - action_fail("Action {} failed: {}".format(action_name, str(e))) - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff --git a/actions/pool-get b/actions/pool-get deleted file mode 100755 index e4f924b..0000000 --- a/actions/pool-get +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/python -__author__ = 'chris' -import sys -from subprocess import check_output, CalledProcessError - -sys.path.append('hooks') - -from charmhelpers.core.hookenv import log, action_set, action_get, action_fail - -if __name__ == '__main__': - name = action_get('pool-name') - key = action_get('key') - try: - out = check_output(['ceph', '--id', 'admin', - 'osd', 'pool', 'get', name, key]).decode('UTF-8') - action_set({'message': out}) - except CalledProcessError as e: - log(e) - action_fail("Pool get failed with message: {}".format(e.message)) diff --git a/actions/pool-set b/actions/pool-set deleted file mode 100755 index 1f6e13b..0000000 --- a/actions/pool-set +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/python -from subprocess import CalledProcessError -import sys - -sys.path.append('hooks') - -from charmhelpers.core.hookenv import action_get, log, action_fail -from ceph_broker import handle_set_pool_value - -if __name__ == '__main__': - name = action_get("pool-name") - key = action_get("key") - value = action_get("value") - request = {'name': name, - 'key': key, - 'value': value} - - try: - handle_set_pool_value(service='admin', request=request) - except CalledProcessError as e: - log(e.message) - action_fail("Setting pool key: {} and value: {} failed with " - "message: {}".format(key, value, e.message)) diff --git a/actions/pool-statistics b/actions/pool-statistics deleted file mode 100755 index 536c889..0000000 --- a/actions/pool-statistics +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/python -import sys - -sys.path.append('hooks') -from subprocess import check_output, CalledProcessError -from charmhelpers.core.hookenv import log, action_set, action_fail - -if __name__ == '__main__': - try: - out = check_output(['ceph', '--id', 'admin', - 'df']).decode('UTF-8') - action_set({'message': out}) - except CalledProcessError as e: - log(e) - action_fail("ceph df failed with message: {}".format(e.message)) diff --git a/actions/remove-pool-snapshot b/actions/remove-pool-snapshot deleted file mode 100755 index 387849e..0000000 --- a/actions/remove-pool-snapshot +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/python -import sys - -sys.path.append('hooks') -from subprocess import CalledProcessError -from charmhelpers.core.hookenv import action_get, log, action_fail -from charmhelpers.contrib.storage.linux.ceph import remove_pool_snapshot - -if __name__ == '__main__': - name = action_get("pool-name") - snapname = action_get("snapshot-name") - try: - remove_pool_snapshot(service='admin', - pool_name=name, - snapshot_name=snapname) - except CalledProcessError as e: - log(e) - action_fail("Remove pool snapshot failed with message: {}".format( - e.message)) diff --git a/actions/rename-pool b/actions/rename-pool deleted file mode 100755 index 6fe088e..0000000 --- a/actions/rename-pool +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/python -import sys - -sys.path.append('hooks') -from subprocess import CalledProcessError -from charmhelpers.core.hookenv import action_get, log, action_fail -from charmhelpers.contrib.storage.linux.ceph import rename_pool - -if __name__ == '__main__': - name = action_get("pool-name") - new_name = action_get("new-name") - try: - rename_pool(service='admin', old_name=name, new_name=new_name) - except CalledProcessError as e: - log(e) - action_fail("Renaming pool failed with message: {}".format(e.message)) diff --git a/actions/resume b/actions/resume deleted file mode 120000 index bd4c0e0..0000000 --- a/actions/resume +++ /dev/null @@ -1 +0,0 @@ -pause_resume.py \ No newline at end of file diff --git a/actions/resume-health b/actions/resume-health deleted file mode 100755 index 39d15a1..0000000 --- a/actions/resume-health +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -set -eux - -ceph osd unset nodown -ceph osd unset noout \ No newline at end of file diff --git a/actions/set-pool-max-bytes b/actions/set-pool-max-bytes deleted file mode 100755 index 8636088..0000000 --- a/actions/set-pool-max-bytes +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/python -import sys - -sys.path.append('hooks') -from subprocess import CalledProcessError -from charmhelpers.core.hookenv import action_get, log, action_fail -from charmhelpers.contrib.storage.linux.ceph import set_pool_quota - -if __name__ == '__main__': - max_bytes = action_get("max") - name = action_get("pool-name") - try: - set_pool_quota(service='admin', pool_name=name, max_bytes=max_bytes) - except CalledProcessError as e: - log(e) - action_fail("Set pool quota failed with message: {}".format(e.message)) diff --git a/actions/snapshot-pool b/actions/snapshot-pool deleted file mode 100755 index a02619b..0000000 --- a/actions/snapshot-pool +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/python -import sys - -sys.path.append('hooks') -from subprocess import CalledProcessError -from charmhelpers.core.hookenv import action_get, log, action_fail -from charmhelpers.contrib.storage.linux.ceph import snapshot_pool - -if __name__ == '__main__': - name = action_get("pool-name") - snapname = action_get("snapshot-name") - try: - snapshot_pool(service='admin', - pool_name=name, - snapshot_name=snapname) - except CalledProcessError as e: - log(e) - action_fail("Snapshot pool failed with message: {}".format(e.message)) diff --git a/charm-helpers-hooks.yaml b/charm-helpers-hooks.yaml deleted file mode 100644 index 6e2e9c0..0000000 --- a/charm-helpers-hooks.yaml +++ /dev/null @@ -1,18 +0,0 @@ -repo: https://github.com/juju/charm-helpers -destination: hooks/charmhelpers -include: - - core - - osplatform - - cli - - fetch - - contrib.python.packages - - contrib.storage.linux - - payload.execd - - contrib.openstack.alternatives - - contrib.network.ip - - contrib.openstack: - - alternatives - - exceptions - - utils - - contrib.charmsupport - - contrib.hardening|inc=* diff --git a/charm-helpers-tests.yaml b/charm-helpers-tests.yaml deleted file mode 100644 index f64f0dd..0000000 --- a/charm-helpers-tests.yaml +++ /dev/null @@ -1,7 +0,0 @@ -repo: https://github.com/juju/charm-helpers -destination: tests/charmhelpers -include: - - contrib.amulet - - contrib.openstack.amulet - - core - - osplatform diff --git a/config.yaml b/config.yaml deleted file mode 100644 index 43784ca..0000000 --- a/config.yaml +++ /dev/null @@ -1,256 +0,0 @@ -options: - loglevel: - default: 1 - type: int - description: Mon and OSD debug level. Max is 20. - fsid: - type: string - default: - description: | - fsid of the ceph cluster. To generate a suitable value use `uuid` - . - This configuration element is mandatory and the service will fail on - install if it is not provided. - config-flags: - type: string - default: - description: | - User provided Ceph configuration. Supports a string representation of - a python dictionary where each top-level key represents a section in - the ceph.conf template. You may only use sections supported in the - template. - . - WARNING: this is not the recommended way to configure the underlying - services that this charm installs and is used at the user's own risk. - This option is mainly provided as a stop-gap for users that either - want to test the effect of modifying some config or who have found - a critical bug in the way the charm has configured their services - and need it fixed immediately. We ask that whenever this is used, - that the user consider opening a bug on this charm at - http://bugs.launchpad.net/charms providing an explanation of why the - config was needed so that we may consider it for inclusion as a - natively supported config in the the charm. - auth-supported: - type: string - default: cephx - description: | - Which authentication flavour to use. - . - Valid options are "cephx" and "none". If "none" is specified, - keys will still be created and deployed so that it can be - enabled later. - monitor-secret: - type: string - default: - description: | - This value will become the mon. key. To generate a suitable value use: - . - ceph-authtool /dev/stdout --name=mon. --gen-key - . - This configuration element is mandatory and the service will fail on - install if it is not provided. - monitor-count: - type: int - default: 3 - description: | - How many nodes to wait for before trying to create the monitor cluster - this number needs to be odd, and more than three is a waste except for - very large clusters. - osd-devices: - type: string - default: /dev/vdb - description: | - The devices to format and set up as osd volumes. - . - These devices are the range of devices that will be checked for and - used across all service units, in addition to any volumes attached - via the --storage flag during deployment. - . - For ceph >= 0.56.6 these can also be directories instead of devices - the - charm assumes anything not starting with /dev is a directory instead. - osd-journal: - type: string - default: - description: | - The device to use as a shared journal drive for all OSD's. By default - no journal device will be used. - . - Only supported with ceph >= 0.48.3. - osd-journal-size: - type: int - default: 1024 - description: | - Ceph osd journal size. The journal size should be at least twice the - product of the expected drive speed multiplied by filestore max sync - interval. However, the most common practice is to partition the journal - drive (often an SSD), and mount it such that Ceph uses the entire - partition for the journal. - . - Only supported with ceph >= 0.48.3. - osd-format: - type: string - default: xfs - description: | - Format of filesystem to use for OSD devices; supported formats include: - . - xfs (Default >= 0.48.3) - ext4 (Only option < 0.48.3) - btrfs (experimental and not recommended) - . - Only supported with ceph >= 0.48.3. - bluestore: - type: boolean - default: false - description: | - Use experimental bluestore storage format for OSD devices; only supported - in Ceph Jewel (10.2.0) or later. - . - Note that despite bluestore being the default for Ceph Luminous, if this - option is False, OSDs will still use filestore. - osd-reformat: - type: string - default: - description: | - By default, the charm will not re-format a device that already looks - as if it might be an OSD device. This is a safeguard to try to - prevent data loss. - . - Specifying this option (any value) forces a reformat of any OSD devices - found which are not already mounted. - ignore-device-errors: - type: boolean - default: False - description: | - By default, the charm will raise errors if a whitelisted device is found, - but for some reason the charm is unable to initialize the device for use - by Ceph. - . - Setting this option to 'True' will result in the charm classifying such - problems as warnings only and will not result in a hook error. - ephemeral-unmount: - type: string - default: - description: | - Cloud instances provider ephermeral storage which is normally mounted - on /mnt. - . - Providing this option will force an unmount of the ephemeral device - so that it can be used as a OSD storage device. This is useful for - testing purposes (cloud deployment is not a typical use case). - source: - type: string - default: - description: | - Optional configuration to support use of additional sources such as: - - - ppa:myteam/ppa - - cloud:trusty-proposed/kilo - - http://my.archive.com/ubuntu main - - The last option should be used in conjunction with the key configuration - option. - - Note that a minimum ceph version of 0.48.2 is required for use with this - charm which is NOT provided by the packages in the main Ubuntu archive - for precise but is provided in the Ubuntu cloud archive. - key: - type: string - default: - description: | - Key ID to import to the apt keyring to support use with arbitary source - configuration from outside of Launchpad archives or PPA's. - use-syslog: - type: boolean - default: False - description: | - If set to True, supporting services will log to syslog. - ceph-public-network: - type: string - default: - description: | - The IP address and netmask of the public (front-side) network (e.g., - 192.168.0.0/24) - . - If multiple networks are to be used, a space-delimited list of a.b.c.d/x - can be provided. - ceph-cluster-network: - type: string - default: - description: | - The IP address and netmask of the cluster (back-side) network (e.g., - 192.168.0.0/24) - . - If multiple networks are to be used, a space-delimited list of a.b.c.d/x - can be provided. - prefer-ipv6: - type: boolean - default: False - description: | - If True enables IPv6 support. The charm will expect network interfaces - to be configured with an IPv6 address. If set to False (default) IPv4 - is expected. - - NOTE: these charms do not currently support IPv6 privacy extension. In - order for this charm to function correctly, the privacy extension must be - disabled and a non-temporary address must be configured/available on - your network interface. - sysctl: - type: string - default: '{ kernel.pid_max : 2097152, vm.max_map_count : 524288, - kernel.threads-max: 2097152 }' - description: | - YAML-formatted associative array of sysctl key/value pairs to be set - persistently. By default we set pid_max, max_map_count and - threads-max to a high value to avoid problems with large numbers (>20) - of OSDs recovering. very large clusters should set those values even - higher (e.g. max for kernel.pid_max is 4194303). - customize-failure-domain: - type: boolean - default: false - description: | - Setting this to true will tell Ceph to replicate across Juju's - Availability Zone instead of specifically by host. - availability_zone: - type: string - default: - description: | - Custom availablility zone to provide to Ceph for the OSD placement - nagios_context: - type: string - default: "juju" - type: string - description: | - Used by the nrpe-external-master subordinate charm. - A string that will be prepended to instance name to set the host name - in nagios. So for instance the hostname would be something like: - juju-myservice-0 - If you're running multiple environments with the same services in them - this allows you to differentiate between them. - nagios_servicegroups: - default: "" - type: string - description: | - A comma-separated list of nagios servicegroups. - If left empty, the nagios_context will be used as the servicegroup - use-direct-io: - default: True - type: boolean - description: Configure use of direct IO for OSD journals. - harden: - default: - type: string - description: | - Apply system hardening. Supports a space-delimited list of modules - to run. Supported modules currently include os, ssh, apache and mysql. - default-rbd-features: - default: - type: int - description: | - Restrict the rbd features used to the specified level. If set, this will - inform clients that they should set the config value `rbd default - features`, for example: - - rbd default features = 1 - - This needs to be set to 1 when deploying a cloud with the nova-lxd - hypervisor. \ No newline at end of file diff --git a/copyright b/copyright deleted file mode 100644 index c801b14..0000000 --- a/copyright +++ /dev/null @@ -1,16 +0,0 @@ -Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0 - -Files: * -Copyright: 2012, Canonical Ltd. -License: Apache-2.0 - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. diff --git a/files/nagios/check_ceph_status.py b/files/nagios/check_ceph_status.py deleted file mode 100755 index cb8d1a1..0000000 --- a/files/nagios/check_ceph_status.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python - -# Copyright (C) 2014 Canonical -# All Rights Reserved -# Author: Jacek Nykis - -import re -import argparse -import subprocess -import nagios_plugin - - -def check_ceph_status(args): - if args.status_file: - nagios_plugin.check_file_freshness(args.status_file, 3600) - with open(args.status_file, "r") as f: - lines = f.readlines() - status_data = dict(l.strip().split(' ', 1) for l in lines if len(l) > 1) - else: - lines = subprocess.check_output(["ceph", "status"]).split('\n') - status_data = dict(l.strip().split(' ', 1) for l in lines if len(l) > 1) - - if ('health' not in status_data - or 'monmap' not in status_data - or 'osdmap'not in status_data): - raise nagios_plugin.UnknownError('UNKNOWN: status data is incomplete') - - if status_data['health'] != 'HEALTH_OK': - msg = 'CRITICAL: ceph health status: "{}"'.format(status_data['health']) - raise nagios_plugin.CriticalError(msg) - osds = re.search("^.*: (\d+) osds: (\d+) up, (\d+) in", status_data['osdmap']) - if osds.group(1) > osds.group(2): # not all OSDs are "up" - msg = 'CRITICAL: Some OSDs are not up. Total: {}, up: {}'.format( - osds.group(1), osds.group(2)) - raise nagios_plugin.CriticalError(msg) - print "All OK" - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Check ceph status') - parser.add_argument('-f', '--file', dest='status_file', - default=False, help='Optional file with "ceph status" output') - args = parser.parse_args() - nagios_plugin.try_check(check_ceph_status, args) diff --git a/files/nagios/collect_ceph_status.sh b/files/nagios/collect_ceph_status.sh deleted file mode 100755 index dbdd3ac..0000000 --- a/files/nagios/collect_ceph_status.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# Copyright (C) 2014 Canonical -# All Rights Reserved -# Author: Jacek Nykis - -LOCK=/var/lock/ceph-status.lock -lockfile-create -r2 --lock-name $LOCK > /dev/null 2>&1 -if [ $? -ne 0 ]; then - exit 1 -fi -trap "rm -f $LOCK > /dev/null 2>&1" exit - -DATA_DIR="/var/lib/nagios" -if [ ! -d $DATA_DIR ]; then - mkdir -p $DATA_DIR -fi - -ceph status >${DATA_DIR}/cat-ceph-status.txt diff --git a/hardening.yaml b/hardening.yaml deleted file mode 100644 index 314bb38..0000000 --- a/hardening.yaml +++ /dev/null @@ -1,5 +0,0 @@ -# Overrides file for contrib.hardening. See README.hardening in -# contrib.hardening for info on how to use this file. -ssh: - server: - use_pam: 'yes' # juju requires this diff --git a/hooks/__init__.py b/hooks/__init__.py deleted file mode 100644 index 9b088de..0000000 --- a/hooks/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/hooks/bootstrap-source-relation-broken b/hooks/bootstrap-source-relation-broken deleted file mode 120000 index 52d9663..0000000 --- a/hooks/bootstrap-source-relation-broken +++ /dev/null @@ -1 +0,0 @@ -ceph_hooks.py \ No newline at end of file diff --git a/hooks/bootstrap-source-relation-changed b/hooks/bootstrap-source-relation-changed deleted file mode 120000 index 52d9663..0000000 --- a/hooks/bootstrap-source-relation-changed +++ /dev/null @@ -1 +0,0 @@ -ceph_hooks.py \ No newline at end of file diff --git a/hooks/bootstrap-source-relation-departed b/hooks/bootstrap-source-relation-departed deleted file mode 120000 index 52d9663..0000000 --- a/hooks/bootstrap-source-relation-departed +++ /dev/null @@ -1 +0,0 @@ -ceph_hooks.py \ No newline at end of file diff --git a/hooks/bootstrap-source-relation-joined b/hooks/bootstrap-source-relation-joined deleted file mode 120000 index 52d9663..0000000 --- a/hooks/bootstrap-source-relation-joined +++ /dev/null @@ -1 +0,0 @@ -ceph_hooks.py \ No newline at end of file diff --git a/hooks/ceph_hooks.py b/hooks/ceph_hooks.py deleted file mode 100755 index 7537df5..0000000 --- a/hooks/ceph_hooks.py +++ /dev/null @@ -1,697 +0,0 @@ -#!/usr/bin/python -# -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys -import socket -import subprocess - -sys.path.append('lib') -import ceph.utils as ceph -from ceph.broker import ( - process_requests -) - -from charmhelpers.core import hookenv -from charmhelpers.core.hookenv import ( - log, - DEBUG, - ERROR, - config, - relation_ids, - related_units, - is_relation_made, - relation_get, - relation_set, - remote_unit, - Hooks, UnregisteredHookError, - service_name, - relations_of_type, - status_set, - storage_get, - storage_list, - local_unit, - application_version_set, -) -from charmhelpers.core.host import ( - service_restart, - service_pause, - umount, - mkdir, - write_file, - rsync, - cmp_pkgrevno, - add_to_updatedb_prunepath, -) -from charmhelpers.fetch import ( - apt_install, - apt_update, - filter_installed_packages, - add_source, - get_upstream_version, -) -from charmhelpers.payload.execd import execd_preinstall -from charmhelpers.contrib.openstack.alternatives import ( - install_alternative, - remove_alternative, -) -from charmhelpers.contrib.network.ip import ( - get_ipv6_addr, - format_ipv6_addr, -) -from charmhelpers.core.sysctl import create as create_sysctl -from charmhelpers.core.templating import render -from charmhelpers.contrib.storage.linux.ceph import ( - CephConfContext, -) -from utils import ( - get_networks, - get_public_addr, - assert_charm_supports_ipv6, - is_unit_paused_set, - get_cluster_addr, -) - -from charmhelpers.contrib.charmsupport import nrpe -from charmhelpers.contrib.hardening.harden import harden - -hooks = Hooks() - -NAGIOS_PLUGINS = '/usr/local/lib/nagios/plugins' -SCRIPTS_DIR = '/usr/local/bin' -STATUS_FILE = '/var/lib/nagios/cat-ceph-status.txt' -STATUS_CRONFILE = '/etc/cron.d/cat-ceph-health' -STORAGE_MOUNT_PATH = '/var/lib/ceph' - - -def check_for_upgrade(): - if not ceph.is_bootstrapped(): - log("Ceph is not bootstrapped, skipping upgrade checks.") - return - - c = hookenv.config() - old_version = ceph.resolve_ceph_version(c.previous('source') or - 'distro') - log('old_version: {}'.format(old_version)) - # Strip all whitespace - new_version = ceph.resolve_ceph_version(hookenv.config('source')) - log('new_version: {}'.format(new_version)) - - if old_version in ceph.UPGRADE_PATHS: - if new_version == ceph.UPGRADE_PATHS[old_version]: - log("{} to {} is a valid upgrade path. Proceeding.".format( - old_version, new_version)) - ceph.roll_monitor_cluster(new_version=new_version, - upgrade_key='admin') - # Wait for all monitors to finish. - status_set("maintenance", "Waiting on mons to finish upgrading") - ceph.wait_for_all_monitors_to_upgrade(new_version=new_version, - upgrade_key='admin') - ceph.roll_osd_cluster(new_version=new_version, - upgrade_key='admin') - else: - # Log a helpful error message - log("Invalid upgrade path from {} to {}. " - "Valid paths are: {}".format( - old_version, - new_version, - ceph.pretty_print_upgrade_paths() - )) - - -@hooks.hook('install.real') -@harden() -def install(): - execd_preinstall() - add_source(config('source'), config('key')) - apt_update(fatal=True) - apt_install(packages=ceph.determine_packages(), fatal=True) - - -def az_info(): - az_info = "" - config_az = config("availability_zone") - juju_az_info = os.environ.get('JUJU_AVAILABILITY_ZONE') - if juju_az_info: - az_info = "{} rack={}".format(az_info, juju_az_info) - if config_az: - az_info = "{} row={}".format(az_info, config_az) - if az_info != "": - log("AZ Info: " + az_info) - return az_info - - -def use_short_objects(): - ''' - Determine whether OSD's should be configured with - limited object name lengths. - - @return: boolean indicating whether OSD's should be limited - ''' - if cmp_pkgrevno('ceph', "10.2.0") >= 0: - if config('osd-format') in ('ext4'): - return True - for device in config('osd-devices'): - if not device.startswith('/dev'): - # TODO: determine format of directory based - # OSD location - return True - return False - - -def get_ceph_context(): - networks = get_networks('ceph-public-network') - public_network = ', '.join(networks) - - networks = get_networks('ceph-cluster-network') - cluster_network = ', '.join(networks) - - cephcontext = { - 'auth_supported': config('auth-supported'), - 'mon_hosts': ' '.join(get_mon_hosts()), - 'fsid': config('fsid'), - 'old_auth': cmp_pkgrevno('ceph', "0.51") < 0, - 'osd_journal_size': config('osd-journal-size'), - 'use_syslog': str(config('use-syslog')).lower(), - 'ceph_public_network': public_network, - 'ceph_cluster_network': cluster_network, - 'loglevel': config('loglevel'), - 'dio': str(config('use-direct-io')).lower(), - 'short_object_len': use_short_objects(), - 'bluestore': config('bluestore'), - 'bluestore_experimental': cmp_pkgrevno('ceph', '12.1.0') < 0, - } - - if config('prefer-ipv6'): - dynamic_ipv6_address = get_ipv6_addr()[0] - if not public_network: - cephcontext['public_addr'] = dynamic_ipv6_address - if not cluster_network: - cephcontext['cluster_addr'] = dynamic_ipv6_address - else: - cephcontext['public_addr'] = get_public_addr() - cephcontext['cluster_addr'] = get_cluster_addr() - - if config('customize-failure-domain'): - az = az_info() - if az: - cephcontext['crush_location'] = "root=default {} host={}" \ - .format(az, socket.gethostname()) - else: - log( - "Your Juju environment doesn't" - "have support for Availability Zones" - ) - - if config('default-rbd-features'): - cephcontext['rbd_features'] = config('default-rbd-features') - - # NOTE(dosaboy): these sections must correspond to what is supported in the - # config template. - sections = ['global', 'mds', 'osd', 'mon'] - cephcontext.update(CephConfContext(permitted_sections=sections)()) - return cephcontext - - -def ceph_conf_path(): - return "/var/lib/charm/{}/ceph.conf".format(service_name()) - - -def emit_cephconf(): - # Install ceph.conf as an alternative to support - # co-existence with other charms that write this file - charm_ceph_conf = ceph_conf_path() - mkdir(os.path.dirname(charm_ceph_conf), owner=ceph.ceph_user(), - group=ceph.ceph_user()) - render('ceph.conf', charm_ceph_conf, get_ceph_context(), perms=0o644) - install_alternative('ceph.conf', '/etc/ceph/ceph.conf', - charm_ceph_conf, 100) - - -JOURNAL_ZAPPED = '/var/lib/ceph/journal_zapped' - - -@hooks.hook('config-changed') -@harden() -def config_changed(): - if config('prefer-ipv6'): - assert_charm_supports_ipv6() - - # Check if an upgrade was requested - check_for_upgrade() - - log('Monitor hosts are ' + repr(get_mon_hosts())) - - # Pre-flight checks - if not config('fsid'): - log('No fsid supplied, cannot proceed.', level=ERROR) - sys.exit(1) - if not config('monitor-secret'): - log('No monitor-secret supplied, cannot proceed.', level=ERROR) - sys.exit(1) - if config('osd-format') not in ceph.DISK_FORMATS: - log('Invalid OSD disk format configuration specified', level=ERROR) - sys.exit(1) - - sysctl_dict = config('sysctl') - if sysctl_dict: - create_sysctl(sysctl_dict, '/etc/sysctl.d/50-ceph-charm.conf') - - emit_cephconf() - - e_mountpoint = config('ephemeral-unmount') - if e_mountpoint and ceph.filesystem_mounted(e_mountpoint): - umount(e_mountpoint) - - osd_journal = get_osd_journal() - if (osd_journal and not os.path.exists(JOURNAL_ZAPPED) and - os.path.exists(osd_journal)): - ceph.zap_disk(osd_journal) - with open(JOURNAL_ZAPPED, 'w') as zapped: - zapped.write('DONE') - - # Support use of single node ceph - if not ceph.is_bootstrapped() and int(config('monitor-count')) == 1: - status_set('maintenance', 'Bootstrapping single Ceph MON') - ceph.bootstrap_monitor_cluster(config('monitor-secret')) - ceph.wait_for_bootstrap() - if cmp_pkgrevno('ceph', '12.0.0') >= 0: - status_set('maintenance', 'Bootstrapping single Ceph MGR') - ceph.bootstrap_manager() - - storage_changed() - - if relations_of_type('nrpe-external-master'): - update_nrpe_config() - add_to_updatedb_prunepath(STORAGE_MOUNT_PATH) - - -@hooks.hook('osd-devices-storage-attached', 'osd-devices-storage-detaching') -def storage_changed(): - if ceph.is_bootstrapped(): - for dev in get_devices(): - ceph.osdize(dev, config('osd-format'), get_osd_journal(), - reformat_osd(), config('ignore-device-errors'), - bluestore=config('bluestore')) - ceph.start_osds(get_devices()) - - -def get_osd_journal(): - ''' - Returns the block device path to use for the OSD journal, if any. - - If there is an osd-journal storage instance attached, it will be - used as the journal. Otherwise, the osd-journal configuration will - be returned. - ''' - storage_ids = storage_list('osd-journal') - if storage_ids: - # There can be at most one osd-journal storage instance. - return storage_get('location', storage_ids[0]) - return config('osd-journal') - - -def get_mon_hosts(): - hosts = [] - addr = get_public_addr() - hosts.append('{}:6789'.format(format_ipv6_addr(addr) or addr)) - - for relid in relation_ids('mon'): - for unit in related_units(relid): - addr = relation_get('ceph-public-address', unit, relid) - if addr is not None: - hosts.append('{}:6789'.format( - format_ipv6_addr(addr) or addr)) - - hosts.sort() - return hosts - - -def get_peer_units(): - """ - Returns a dictionary of unit names from the mon peer relation with - a flag indicating whether the unit has presented its address - """ - units = {} - units[local_unit()] = True - for relid in relation_ids('mon'): - for unit in related_units(relid): - addr = relation_get('ceph-public-address', unit, relid) - units[unit] = addr is not None - return units - - -def reformat_osd(): - if config('osd-reformat'): - return True - else: - return False - - -def get_devices(): - devices = [] - - if config('osd-devices'): - for path in config('osd-devices').split(' '): - path = path.strip() - # Make sure its a device which is specified using an - # absolute path so that the current working directory - # or any relative path under this directory is not used - if os.path.isabs(path): - devices.append(os.path.realpath(path)) - - # List storage instances for the 'osd-devices' - # store declared for this charm too, and add - # their block device paths to the list. - storage_ids = storage_list('osd-devices') - devices.extend((storage_get('location', s) for s in storage_ids)) - return devices - - -@hooks.hook('mon-relation-joined') -def mon_relation_joined(): - public_addr = get_public_addr() - for relid in relation_ids('mon'): - relation_set(relation_id=relid, - relation_settings={'ceph-public-address': public_addr}) - - -@hooks.hook('mon-relation-departed', - 'mon-relation-changed') -def mon_relation(): - emit_cephconf() - - moncount = int(config('monitor-count')) - if len(get_mon_hosts()) >= moncount: - status_set('maintenance', 'Bootstrapping MON cluster') - ceph.bootstrap_monitor_cluster(config('monitor-secret')) - ceph.wait_for_bootstrap() - if cmp_pkgrevno('ceph', '12.0.0') >= 0: - status_set('maintenance', 'Bootstrapping Ceph MGR') - ceph.bootstrap_manager() - for dev in get_devices(): - ceph.osdize(dev, config('osd-format'), get_osd_journal(), - reformat_osd(), config('ignore-device-errors'), - bluestore=config('bluestore')) - ceph.start_osds(get_devices()) - ceph.wait_for_quorum() - notify_osds() - notify_radosgws() - notify_client() - else: - log('Not enough mons ({}), punting.' - .format(len(get_mon_hosts()))) - - -def notify_osds(): - for relid in relation_ids('osd'): - osd_relation(relid) - - -def notify_radosgws(): - for relid in relation_ids('radosgw'): - for unit in related_units(relid): - radosgw_relation(relid=relid, unit=unit) - - -def notify_client(): - for relid in relation_ids('client'): - client_relation_joined(relid) - for unit in related_units(relid): - client_relation_changed(relid, unit) - - -@hooks.hook('osd-relation-changed') -@hooks.hook('osd-relation-joined') -def osd_relation(relid=None): - if ceph.is_quorum(): - log('mon cluster in quorum - providing fsid & keys') - public_addr = get_public_addr() - data = { - 'fsid': config('fsid'), - 'osd_bootstrap_key': ceph.get_osd_bootstrap_key(), - 'auth': config('auth-supported'), - 'ceph-public-address': public_addr, - 'osd_upgrade_key': ceph.get_named_key('osd-upgrade', - caps=ceph.osd_upgrade_caps), - } - - unit = remote_unit() - settings = relation_get(rid=relid, unit=unit) - """Process broker request(s).""" - if 'broker_req' in settings: - if ceph.is_leader(): - rsp = process_requests(settings['broker_req']) - unit_id = unit.replace('/', '-') - unit_response_key = 'broker-rsp-' + unit_id - data[unit_response_key] = rsp - else: - log("Not leader - ignoring broker request", level=DEBUG) - - relation_set(relation_id=relid, - relation_settings=data) - else: - log('mon cluster not in quorum - deferring fsid provision') - - -@hooks.hook('radosgw-relation-changed') -@hooks.hook('radosgw-relation-joined') -def radosgw_relation(relid=None, unit=None): - # Install radosgw for admin tools - apt_install(packages=filter_installed_packages(['radosgw'])) - if not unit: - unit = remote_unit() - - if ceph.is_quorum(): - log('mon cluster in quorum - providing radosgw with keys') - public_addr = get_public_addr() - data = { - 'fsid': config('fsid'), - 'radosgw_key': ceph.get_radosgw_key(), - 'auth': config('auth-supported'), - 'ceph-public-address': public_addr, - } - - settings = relation_get(rid=relid, unit=unit) - """Process broker request(s).""" - if 'broker_req' in settings: - if ceph.is_leader(): - rsp = process_requests(settings['broker_req']) - unit_id = unit.replace('/', '-') - unit_response_key = 'broker-rsp-' + unit_id - data[unit_response_key] = rsp - else: - log("Not leader - ignoring broker request", level=DEBUG) - - relation_set(relation_id=relid, relation_settings=data) - else: - log('mon cluster not in quorum - deferring key provision') - - -@hooks.hook('client-relation-joined') -def client_relation_joined(relid=None): - if ceph.is_quorum(): - log('mon cluster in quorum - providing client with keys') - service_name = None - if relid is None: - units = [remote_unit()] - service_name = units[0].split('/')[0] - else: - units = related_units(relid) - if len(units) > 0: - service_name = units[0].split('/')[0] - - if service_name is not None: - public_addr = get_public_addr() - data = {'key': ceph.get_named_key(service_name), - 'auth': config('auth-supported'), - 'ceph-public-address': public_addr} - if config('default-rbd-features'): - data['rbd-features'] = config('default-rbd-features') - relation_set(relation_id=relid, - relation_settings=data) - else: - log('mon cluster not in quorum - deferring key provision') - - -@hooks.hook('client-relation-changed') -def client_relation_changed(relid=None, unit=None): - """Process broker requests from ceph client relations.""" - if ceph.is_quorum(): - if not unit: - unit = remote_unit() - settings = relation_get(rid=relid, unit=unit) - if 'broker_req' in settings: - if not ceph.is_leader(): - log("Not leader - ignoring broker request", level=DEBUG) - else: - rsp = process_requests(settings['broker_req']) - unit_id = remote_unit().replace('/', '-') - unit_response_key = 'broker-rsp-' + unit_id - # broker_rsp is being left for backward compatibility, - # unit_response_key superscedes it - data = { - 'broker_rsp': rsp, - unit_response_key: rsp, - } - relation_set(relation_id=relid, - relation_settings=data) - else: - log('mon cluster not in quorum', level=DEBUG) - - -@hooks.hook('bootstrap-source-relation-joined') -def bootstrap_source_joined(relid=None): - """Provide required information to bootstrap ceph-mon cluster""" - if ceph.is_quorum(): - source = { - 'fsid': config('fsid'), - 'monitor-secret': config('monitor-secret'), - 'ceph-public-address': get_public_addr(), - } - relation_set(relation_id=relid, - relation_settings=source) - - -@hooks.hook('upgrade-charm.real') -@harden() -def upgrade_charm(): - emit_cephconf() - apt_install(packages=filter_installed_packages(ceph.determine_packages()), - fatal=True) - ceph.update_monfs() - mon_relation_joined() - if is_relation_made("nrpe-external-master"): - update_nrpe_config() - - -@hooks.hook('start') -def start(): - # In case we're being redeployed to the same machines, try - # to make sure everything is running as soon as possible. - if ceph.systemd(): - service_restart('ceph-mon') - else: - service_restart('ceph-mon-all') - if cmp_pkgrevno('ceph', '12.0.0') >= 0: - service_restart('ceph-mgr@{}'.format(socket.gethostname())) - if ceph.is_bootstrapped(): - ceph.start_osds(get_devices()) - - -@hooks.hook('nrpe-external-master-relation-joined') -@hooks.hook('nrpe-external-master-relation-changed') -def update_nrpe_config(): - # python-dbus is used by check_upstart_job - # lockfile-create is used by collect_ceph_status - apt_install(['python-dbus', 'lockfile-progs']) - log('Refreshing nagios checks') - if os.path.isdir(NAGIOS_PLUGINS): - rsync(os.path.join(os.getenv('CHARM_DIR'), 'files', 'nagios', - 'check_ceph_status.py'), - os.path.join(NAGIOS_PLUGINS, 'check_ceph_status.py')) - - script = os.path.join(SCRIPTS_DIR, 'collect_ceph_status.sh') - rsync(os.path.join(os.getenv('CHARM_DIR'), 'files', - 'nagios', 'collect_ceph_status.sh'), - script) - cronjob = "{} root {}\n".format('*/5 * * * *', script) - write_file(STATUS_CRONFILE, cronjob) - - # Find out if nrpe set nagios_hostname - hostname = nrpe.get_nagios_hostname() - current_unit = nrpe.get_nagios_unit_name() - nrpe_setup = nrpe.NRPE(hostname=hostname) - nrpe_setup.add_check( - shortname="ceph", - description='Check Ceph health {%s}' % current_unit, - check_cmd='check_ceph_status.py -f {}'.format(STATUS_FILE) - ) - nrpe_setup.write() - - -VERSION_PACKAGE = 'ceph-common' - - -def assess_status(): - """Assess status of current unit""" - application_version_set(get_upstream_version(VERSION_PACKAGE)) - # check to see if the unit is paused. - if is_unit_paused_set(): - status_set('maintenance', - "Paused. Use 'resume' action to resume normal service.") - return - moncount = int(config('monitor-count')) - units = get_peer_units() - # not enough peers and mon_count > 1 - if len(units.keys()) < moncount: - status_set('blocked', 'Insufficient peer units to bootstrap' - ' cluster (require {})'.format(moncount)) - return - - # mon_count > 1, peers, but no ceph-public-address - ready = sum(1 for unit_ready in units.itervalues() if unit_ready) - if ready < moncount: - status_set('waiting', 'Peer units detected, waiting for addresses') - return - - # active - bootstrapped + quorum status check - if ceph.is_bootstrapped() and ceph.is_quorum(): - status_set('active', 'Unit is ready and clustered') - else: - # Unit should be running and clustered, but no quorum - # TODO: should this be blocked or waiting? - status_set('blocked', 'Unit not clustered (no quorum)') - # If there's a pending lock for this unit, - # can i get the lock? - # reboot the ceph-mon process - - -@hooks.hook('update-status') -@harden() -def update_status(): - log('Updating status.') - - -@hooks.hook('stop') -def stop(): - # NOTE(jamespage) - # Ensure monitor is removed from monmap prior to shutdown - # otherwise we end up with odd quorum loss issues during - # migration. - # NOTE(jamespage): remove is compat with >= firefly - cmd = ['ceph', 'mon', 'remove', socket.gethostname()] - subprocess.check_call(cmd) - # NOTE(jamespage) - # Pause MON and MGR processes running on this unit, leaving - # any OSD processes running, supporting the migration to - # using the ceph-mon charm. - service_pause('ceph-mon') - if cmp_pkgrevno('ceph', '12.0.0') >= 0: - service_pause('ceph-mgr@{}'.format(socket.gethostname())) - # NOTE(jamespage) - # Remove the ceph.conf provided by this charm so - # that the ceph.conf from other deployed applications - # can take priority post removal. - remove_alternative('ceph.conf', ceph_conf_path()) - - -if __name__ == '__main__': - try: - hooks.execute(sys.argv) - except UnregisteredHookError as e: - log('Unknown hook {} - skipping.'.format(e)) - assess_status() diff --git a/hooks/charmhelpers/__init__.py b/hooks/charmhelpers/__init__.py deleted file mode 100644 index e7aa471..0000000 --- a/hooks/charmhelpers/__init__.py +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Bootstrap charm-helpers, installing its dependencies if necessary using -# only standard libraries. -from __future__ import print_function -from __future__ import absolute_import - -import functools -import inspect -import subprocess -import sys - -try: - import six # flake8: noqa -except ImportError: - if sys.version_info.major == 2: - subprocess.check_call(['apt-get', 'install', '-y', 'python-six']) - else: - subprocess.check_call(['apt-get', 'install', '-y', 'python3-six']) - import six # flake8: noqa - -try: - import yaml # flake8: noqa -except ImportError: - if sys.version_info.major == 2: - subprocess.check_call(['apt-get', 'install', '-y', 'python-yaml']) - else: - subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml']) - import yaml # flake8: noqa - - -# Holds a list of mapping of mangled function names that have been deprecated -# using the @deprecate decorator below. This is so that the warning is only -# printed once for each usage of the function. -__deprecated_functions = {} - - -def deprecate(warning, date=None, log=None): - """Add a deprecation warning the first time the function is used. - The date, which is a string in semi-ISO8660 format indicate the year-month - that the function is officially going to be removed. - - usage: - - @deprecate('use core/fetch/add_source() instead', '2017-04') - def contributed_add_source_thing(...): - ... - - And it then prints to the log ONCE that the function is deprecated. - The reason for passing the logging function (log) is so that hookenv.log - can be used for a charm if needed. - - :param warning: String to indicat where it has moved ot. - :param date: optional sting, in YYYY-MM format to indicate when the - function will definitely (probably) be removed. - :param log: The log function to call to log. If not, logs to stdout - """ - def wrap(f): - - @functools.wraps(f) - def wrapped_f(*args, **kwargs): - try: - module = inspect.getmodule(f) - file = inspect.getsourcefile(f) - lines = inspect.getsourcelines(f) - f_name = "{}-{}-{}..{}-{}".format( - module.__name__, file, lines[0], lines[-1], f.__name__) - except (IOError, TypeError): - # assume it was local, so just use the name of the function - f_name = f.__name__ - if f_name not in __deprecated_functions: - __deprecated_functions[f_name] = True - s = "DEPRECATION WARNING: Function {} is being removed".format( - f.__name__) - if date: - s = "{} on/around {}".format(s, date) - if warning: - s = "{} : {}".format(s, warning) - if log: - log(s) - else: - print(s) - return f(*args, **kwargs) - return wrapped_f - return wrap diff --git a/hooks/charmhelpers/cli/__init__.py b/hooks/charmhelpers/cli/__init__.py deleted file mode 100644 index 389b490..0000000 --- a/hooks/charmhelpers/cli/__init__.py +++ /dev/null @@ -1,189 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import inspect -import argparse -import sys - -from six.moves import zip - -import charmhelpers.core.unitdata - - -class OutputFormatter(object): - def __init__(self, outfile=sys.stdout): - self.formats = ( - "raw", - "json", - "py", - "yaml", - "csv", - "tab", - ) - self.outfile = outfile - - def add_arguments(self, argument_parser): - formatgroup = argument_parser.add_mutually_exclusive_group() - choices = self.supported_formats - formatgroup.add_argument("--format", metavar='FMT', - help="Select output format for returned data, " - "where FMT is one of: {}".format(choices), - choices=choices, default='raw') - for fmt in self.formats: - fmtfunc = getattr(self, fmt) - formatgroup.add_argument("-{}".format(fmt[0]), - "--{}".format(fmt), action='store_const', - const=fmt, dest='format', - help=fmtfunc.__doc__) - - @property - def supported_formats(self): - return self.formats - - def raw(self, output): - """Output data as raw string (default)""" - if isinstance(output, (list, tuple)): - output = '\n'.join(map(str, output)) - self.outfile.write(str(output)) - - def py(self, output): - """Output data as a nicely-formatted python data structure""" - import pprint - pprint.pprint(output, stream=self.outfile) - - def json(self, output): - """Output data in JSON format""" - import json - json.dump(output, self.outfile) - - def yaml(self, output): - """Output data in YAML format""" - import yaml - yaml.safe_dump(output, self.outfile) - - def csv(self, output): - """Output data as excel-compatible CSV""" - import csv - csvwriter = csv.writer(self.outfile) - csvwriter.writerows(output) - - def tab(self, output): - """Output data in excel-compatible tab-delimited format""" - import csv - csvwriter = csv.writer(self.outfile, dialect=csv.excel_tab) - csvwriter.writerows(output) - - def format_output(self, output, fmt='raw'): - fmtfunc = getattr(self, fmt) - fmtfunc(output) - - -class CommandLine(object): - argument_parser = None - subparsers = None - formatter = None - exit_code = 0 - - def __init__(self): - if not self.argument_parser: - self.argument_parser = argparse.ArgumentParser(description='Perform common charm tasks') - if not self.formatter: - self.formatter = OutputFormatter() - self.formatter.add_arguments(self.argument_parser) - if not self.subparsers: - self.subparsers = self.argument_parser.add_subparsers(help='Commands') - - def subcommand(self, command_name=None): - """ - Decorate a function as a subcommand. Use its arguments as the - command-line arguments""" - def wrapper(decorated): - cmd_name = command_name or decorated.__name__ - subparser = self.subparsers.add_parser(cmd_name, - description=decorated.__doc__) - for args, kwargs in describe_arguments(decorated): - subparser.add_argument(*args, **kwargs) - subparser.set_defaults(func=decorated) - return decorated - return wrapper - - def test_command(self, decorated): - """ - Subcommand is a boolean test function, so bool return values should be - converted to a 0/1 exit code. - """ - decorated._cli_test_command = True - return decorated - - def no_output(self, decorated): - """ - Subcommand is not expected to return a value, so don't print a spurious None. - """ - decorated._cli_no_output = True - return decorated - - def subcommand_builder(self, command_name, description=None): - """ - Decorate a function that builds a subcommand. Builders should accept a - single argument (the subparser instance) and return the function to be - run as the command.""" - def wrapper(decorated): - subparser = self.subparsers.add_parser(command_name) - func = decorated(subparser) - subparser.set_defaults(func=func) - subparser.description = description or func.__doc__ - return wrapper - - def run(self): - "Run cli, processing arguments and executing subcommands." - arguments = self.argument_parser.parse_args() - argspec = inspect.getargspec(arguments.func) - vargs = [] - for arg in argspec.args: - vargs.append(getattr(arguments, arg)) - if argspec.varargs: - vargs.extend(getattr(arguments, argspec.varargs)) - output = arguments.func(*vargs) - if getattr(arguments.func, '_cli_test_command', False): - self.exit_code = 0 if output else 1 - output = '' - if getattr(arguments.func, '_cli_no_output', False): - output = '' - self.formatter.format_output(output, arguments.format) - if charmhelpers.core.unitdata._KV: - charmhelpers.core.unitdata._KV.flush() - - -cmdline = CommandLine() - - -def describe_arguments(func): - """ - Analyze a function's signature and return a data structure suitable for - passing in as arguments to an argparse parser's add_argument() method.""" - - argspec = inspect.getargspec(func) - # we should probably raise an exception somewhere if func includes **kwargs - if argspec.defaults: - positional_args = argspec.args[:-len(argspec.defaults)] - keyword_names = argspec.args[-len(argspec.defaults):] - for arg, default in zip(keyword_names, argspec.defaults): - yield ('--{}'.format(arg),), {'default': default} - else: - positional_args = argspec.args - - for arg in positional_args: - yield (arg,), {} - if argspec.varargs: - yield (argspec.varargs,), {'nargs': '*'} diff --git a/hooks/charmhelpers/cli/benchmark.py b/hooks/charmhelpers/cli/benchmark.py deleted file mode 100644 index 303af14..0000000 --- a/hooks/charmhelpers/cli/benchmark.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from . import cmdline -from charmhelpers.contrib.benchmark import Benchmark - - -@cmdline.subcommand(command_name='benchmark-start') -def start(): - Benchmark.start() - - -@cmdline.subcommand(command_name='benchmark-finish') -def finish(): - Benchmark.finish() - - -@cmdline.subcommand_builder('benchmark-composite', description="Set the benchmark composite score") -def service(subparser): - subparser.add_argument("value", help="The composite score.") - subparser.add_argument("units", help="The units the composite score represents, i.e., 'reads/sec'.") - subparser.add_argument("direction", help="'asc' if a lower score is better, 'desc' if a higher score is better.") - return Benchmark.set_composite_score diff --git a/hooks/charmhelpers/cli/commands.py b/hooks/charmhelpers/cli/commands.py deleted file mode 100644 index b931056..0000000 --- a/hooks/charmhelpers/cli/commands.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -This module loads sub-modules into the python runtime so they can be -discovered via the inspect module. In order to prevent flake8 from (rightfully) -telling us these are unused modules, throw a ' # noqa' at the end of each import -so that the warning is suppressed. -""" - -from . import CommandLine # noqa - -""" -Import the sub-modules which have decorated subcommands to register with chlp. -""" -from . import host # noqa -from . import benchmark # noqa -from . import unitdata # noqa -from . import hookenv # noqa diff --git a/hooks/charmhelpers/cli/hookenv.py b/hooks/charmhelpers/cli/hookenv.py deleted file mode 100644 index bd72f44..0000000 --- a/hooks/charmhelpers/cli/hookenv.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from . import cmdline -from charmhelpers.core import hookenv - - -cmdline.subcommand('relation-id')(hookenv.relation_id._wrapped) -cmdline.subcommand('service-name')(hookenv.service_name) -cmdline.subcommand('remote-service-name')(hookenv.remote_service_name._wrapped) diff --git a/hooks/charmhelpers/cli/host.py b/hooks/charmhelpers/cli/host.py deleted file mode 100644 index 4039684..0000000 --- a/hooks/charmhelpers/cli/host.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from . import cmdline -from charmhelpers.core import host - - -@cmdline.subcommand() -def mounts(): - "List mounts" - return host.mounts() - - -@cmdline.subcommand_builder('service', description="Control system services") -def service(subparser): - subparser.add_argument("action", help="The action to perform (start, stop, etc...)") - subparser.add_argument("service_name", help="Name of the service to control") - return host.service diff --git a/hooks/charmhelpers/cli/unitdata.py b/hooks/charmhelpers/cli/unitdata.py deleted file mode 100644 index c572858..0000000 --- a/hooks/charmhelpers/cli/unitdata.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from . import cmdline -from charmhelpers.core import unitdata - - -@cmdline.subcommand_builder('unitdata', description="Store and retrieve data") -def unitdata_cmd(subparser): - nested = subparser.add_subparsers() - get_cmd = nested.add_parser('get', help='Retrieve data') - get_cmd.add_argument('key', help='Key to retrieve the value of') - get_cmd.set_defaults(action='get', value=None) - set_cmd = nested.add_parser('set', help='Store data') - set_cmd.add_argument('key', help='Key to set') - set_cmd.add_argument('value', help='Value to store') - set_cmd.set_defaults(action='set') - - def _unitdata_cmd(action, key, value): - if action == 'get': - return unitdata.kv().get(key) - elif action == 'set': - unitdata.kv().set(key, value) - unitdata.kv().flush() - return '' - return _unitdata_cmd diff --git a/hooks/charmhelpers/contrib/__init__.py b/hooks/charmhelpers/contrib/__init__.py deleted file mode 100644 index d7567b8..0000000 --- a/hooks/charmhelpers/contrib/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/hooks/charmhelpers/contrib/charmsupport/__init__.py b/hooks/charmhelpers/contrib/charmsupport/__init__.py deleted file mode 100644 index d7567b8..0000000 --- a/hooks/charmhelpers/contrib/charmsupport/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/hooks/charmhelpers/contrib/charmsupport/nrpe.py b/hooks/charmhelpers/contrib/charmsupport/nrpe.py deleted file mode 100644 index 1c55b30..0000000 --- a/hooks/charmhelpers/contrib/charmsupport/nrpe.py +++ /dev/null @@ -1,445 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Compatibility with the nrpe-external-master charm""" -# Copyright 2012 Canonical Ltd. -# -# Authors: -# Matthew Wedgwood - -import subprocess -import pwd -import grp -import os -import glob -import shutil -import re -import shlex -import yaml - -from charmhelpers.core.hookenv import ( - config, - hook_name, - local_unit, - log, - relation_ids, - relation_set, - relations_of_type, -) - -from charmhelpers.core.host import service -from charmhelpers.core import host - -# This module adds compatibility with the nrpe-external-master and plain nrpe -# subordinate charms. To use it in your charm: -# -# 1. Update metadata.yaml -# -# provides: -# (...) -# nrpe-external-master: -# interface: nrpe-external-master -# scope: container -# -# and/or -# -# provides: -# (...) -# local-monitors: -# interface: local-monitors -# scope: container - -# -# 2. Add the following to config.yaml -# -# nagios_context: -# default: "juju" -# type: string -# description: | -# Used by the nrpe subordinate charms. -# A string that will be prepended to instance name to set the host name -# in nagios. So for instance the hostname would be something like: -# juju-myservice-0 -# If you're running multiple environments with the same services in them -# this allows you to differentiate between them. -# nagios_servicegroups: -# default: "" -# type: string -# description: | -# A comma-separated list of nagios servicegroups. -# If left empty, the nagios_context will be used as the servicegroup -# -# 3. Add custom checks (Nagios plugins) to files/nrpe-external-master -# -# 4. Update your hooks.py with something like this: -# -# from charmsupport.nrpe import NRPE -# (...) -# def update_nrpe_config(): -# nrpe_compat = NRPE() -# nrpe_compat.add_check( -# shortname = "myservice", -# description = "Check MyService", -# check_cmd = "check_http -w 2 -c 10 http://localhost" -# ) -# nrpe_compat.add_check( -# "myservice_other", -# "Check for widget failures", -# check_cmd = "/srv/myapp/scripts/widget_check" -# ) -# nrpe_compat.write() -# -# def config_changed(): -# (...) -# update_nrpe_config() -# -# def nrpe_external_master_relation_changed(): -# update_nrpe_config() -# -# def local_monitors_relation_changed(): -# update_nrpe_config() -# -# 4.a If your charm is a subordinate charm set primary=False -# -# from charmsupport.nrpe import NRPE -# (...) -# def update_nrpe_config(): -# nrpe_compat = NRPE(primary=False) -# -# 5. ln -s hooks.py nrpe-external-master-relation-changed -# ln -s hooks.py local-monitors-relation-changed - - -class CheckException(Exception): - pass - - -class Check(object): - shortname_re = '[A-Za-z0-9-_.]+$' - service_template = (""" -#--------------------------------------------------- -# This file is Juju managed -#--------------------------------------------------- -define service {{ - use active-service - host_name {nagios_hostname} - service_description {nagios_hostname}[{shortname}] """ - """{description} - check_command check_nrpe!{command} - servicegroups {nagios_servicegroup} -}} -""") - - def __init__(self, shortname, description, check_cmd): - super(Check, self).__init__() - # XXX: could be better to calculate this from the service name - if not re.match(self.shortname_re, shortname): - raise CheckException("shortname must match {}".format( - Check.shortname_re)) - self.shortname = shortname - self.command = "check_{}".format(shortname) - # Note: a set of invalid characters is defined by the - # Nagios server config - # The default is: illegal_object_name_chars=`~!$%^&*"|'<>?,()= - self.description = description - self.check_cmd = self._locate_cmd(check_cmd) - - def _get_check_filename(self): - return os.path.join(NRPE.nrpe_confdir, '{}.cfg'.format(self.command)) - - def _get_service_filename(self, hostname): - return os.path.join(NRPE.nagios_exportdir, - 'service__{}_{}.cfg'.format(hostname, self.command)) - - def _locate_cmd(self, check_cmd): - search_path = ( - '/usr/lib/nagios/plugins', - '/usr/local/lib/nagios/plugins', - ) - parts = shlex.split(check_cmd) - for path in search_path: - if os.path.exists(os.path.join(path, parts[0])): - command = os.path.join(path, parts[0]) - if len(parts) > 1: - command += " " + " ".join(parts[1:]) - return command - log('Check command not found: {}'.format(parts[0])) - return '' - - def _remove_service_files(self): - if not os.path.exists(NRPE.nagios_exportdir): - return - for f in os.listdir(NRPE.nagios_exportdir): - if f.endswith('_{}.cfg'.format(self.command)): - os.remove(os.path.join(NRPE.nagios_exportdir, f)) - - def remove(self, hostname): - nrpe_check_file = self._get_check_filename() - if os.path.exists(nrpe_check_file): - os.remove(nrpe_check_file) - self._remove_service_files() - - def write(self, nagios_context, hostname, nagios_servicegroups): - nrpe_check_file = self._get_check_filename() - with open(nrpe_check_file, 'w') as nrpe_check_config: - nrpe_check_config.write("# check {}\n".format(self.shortname)) - if nagios_servicegroups: - nrpe_check_config.write( - "# The following header was added automatically by juju\n") - nrpe_check_config.write( - "# Modifying it will affect nagios monitoring and alerting\n") - nrpe_check_config.write( - "# servicegroups: {}\n".format(nagios_servicegroups)) - nrpe_check_config.write("command[{}]={}\n".format( - self.command, self.check_cmd)) - - if not os.path.exists(NRPE.nagios_exportdir): - log('Not writing service config as {} is not accessible'.format( - NRPE.nagios_exportdir)) - else: - self.write_service_config(nagios_context, hostname, - nagios_servicegroups) - - def write_service_config(self, nagios_context, hostname, - nagios_servicegroups): - self._remove_service_files() - - templ_vars = { - 'nagios_hostname': hostname, - 'nagios_servicegroup': nagios_servicegroups, - 'description': self.description, - 'shortname': self.shortname, - 'command': self.command, - } - nrpe_service_text = Check.service_template.format(**templ_vars) - nrpe_service_file = self._get_service_filename(hostname) - with open(nrpe_service_file, 'w') as nrpe_service_config: - nrpe_service_config.write(str(nrpe_service_text)) - - def run(self): - subprocess.call(self.check_cmd) - - -class NRPE(object): - nagios_logdir = '/var/log/nagios' - nagios_exportdir = '/var/lib/nagios/export' - nrpe_confdir = '/etc/nagios/nrpe.d' - homedir = '/var/lib/nagios' # home dir provided by nagios-nrpe-server - - def __init__(self, hostname=None, primary=True): - super(NRPE, self).__init__() - self.config = config() - self.primary = primary - self.nagios_context = self.config['nagios_context'] - if 'nagios_servicegroups' in self.config and self.config['nagios_servicegroups']: - self.nagios_servicegroups = self.config['nagios_servicegroups'] - else: - self.nagios_servicegroups = self.nagios_context - self.unit_name = local_unit().replace('/', '-') - if hostname: - self.hostname = hostname - else: - nagios_hostname = get_nagios_hostname() - if nagios_hostname: - self.hostname = nagios_hostname - else: - self.hostname = "{}-{}".format(self.nagios_context, self.unit_name) - self.checks = [] - # Iff in an nrpe-external-master relation hook, set primary status - relation = relation_ids('nrpe-external-master') - if relation: - log("Setting charm primary status {}".format(primary)) - for rid in relation_ids('nrpe-external-master'): - relation_set(relation_id=rid, relation_settings={'primary': self.primary}) - - def add_check(self, *args, **kwargs): - self.checks.append(Check(*args, **kwargs)) - - def remove_check(self, *args, **kwargs): - if kwargs.get('shortname') is None: - raise ValueError('shortname of check must be specified') - - # Use sensible defaults if they're not specified - these are not - # actually used during removal, but they're required for constructing - # the Check object; check_disk is chosen because it's part of the - # nagios-plugins-basic package. - if kwargs.get('check_cmd') is None: - kwargs['check_cmd'] = 'check_disk' - if kwargs.get('description') is None: - kwargs['description'] = '' - - check = Check(*args, **kwargs) - check.remove(self.hostname) - - def write(self): - try: - nagios_uid = pwd.getpwnam('nagios').pw_uid - nagios_gid = grp.getgrnam('nagios').gr_gid - except Exception: - log("Nagios user not set up, nrpe checks not updated") - return - - if not os.path.exists(NRPE.nagios_logdir): - os.mkdir(NRPE.nagios_logdir) - os.chown(NRPE.nagios_logdir, nagios_uid, nagios_gid) - - nrpe_monitors = {} - monitors = {"monitors": {"remote": {"nrpe": nrpe_monitors}}} - for nrpecheck in self.checks: - nrpecheck.write(self.nagios_context, self.hostname, - self.nagios_servicegroups) - nrpe_monitors[nrpecheck.shortname] = { - "command": nrpecheck.command, - } - - # update-status hooks are configured to firing every 5 minutes by - # default. When nagios-nrpe-server is restarted, the nagios server - # reports checks failing causing unneccessary alerts. Let's not restart - # on update-status hooks. - if not hook_name() == 'update-status': - service('restart', 'nagios-nrpe-server') - - monitor_ids = relation_ids("local-monitors") + \ - relation_ids("nrpe-external-master") - for rid in monitor_ids: - relation_set(relation_id=rid, monitors=yaml.dump(monitors)) - - -def get_nagios_hostcontext(relation_name='nrpe-external-master'): - """ - Query relation with nrpe subordinate, return the nagios_host_context - - :param str relation_name: Name of relation nrpe sub joined to - """ - for rel in relations_of_type(relation_name): - if 'nagios_host_context' in rel: - return rel['nagios_host_context'] - - -def get_nagios_hostname(relation_name='nrpe-external-master'): - """ - Query relation with nrpe subordinate, return the nagios_hostname - - :param str relation_name: Name of relation nrpe sub joined to - """ - for rel in relations_of_type(relation_name): - if 'nagios_hostname' in rel: - return rel['nagios_hostname'] - - -def get_nagios_unit_name(relation_name='nrpe-external-master'): - """ - Return the nagios unit name prepended with host_context if needed - - :param str relation_name: Name of relation nrpe sub joined to - """ - host_context = get_nagios_hostcontext(relation_name) - if host_context: - unit = "%s:%s" % (host_context, local_unit()) - else: - unit = local_unit() - return unit - - -def add_init_service_checks(nrpe, services, unit_name, immediate_check=True): - """ - Add checks for each service in list - - :param NRPE nrpe: NRPE object to add check to - :param list services: List of services to check - :param str unit_name: Unit name to use in check description - :param bool immediate_check: For sysv init, run the service check immediately - """ - for svc in services: - # Don't add a check for these services from neutron-gateway - if svc in ['ext-port', 'os-charm-phy-nic-mtu']: - next - - upstart_init = '/etc/init/%s.conf' % svc - sysv_init = '/etc/init.d/%s' % svc - - if host.init_is_systemd(): - nrpe.add_check( - shortname=svc, - description='process check {%s}' % unit_name, - check_cmd='check_systemd.py %s' % svc - ) - elif os.path.exists(upstart_init): - nrpe.add_check( - shortname=svc, - description='process check {%s}' % unit_name, - check_cmd='check_upstart_job %s' % svc - ) - elif os.path.exists(sysv_init): - cronpath = '/etc/cron.d/nagios-service-check-%s' % svc - checkpath = '%s/service-check-%s.txt' % (nrpe.homedir, svc) - croncmd = ( - '/usr/local/lib/nagios/plugins/check_exit_status.pl ' - '-e -s /etc/init.d/%s status' % svc - ) - cron_file = '*/5 * * * * root %s > %s\n' % (croncmd, checkpath) - f = open(cronpath, 'w') - f.write(cron_file) - f.close() - nrpe.add_check( - shortname=svc, - description='service check {%s}' % unit_name, - check_cmd='check_status_file.py -f %s' % checkpath, - ) - # if /var/lib/nagios doesn't exist open(checkpath, 'w') will fail - # (LP: #1670223). - if immediate_check and os.path.isdir(nrpe.homedir): - f = open(checkpath, 'w') - subprocess.call( - croncmd.split(), - stdout=f, - stderr=subprocess.STDOUT - ) - f.close() - os.chmod(checkpath, 0o644) - - -def copy_nrpe_checks(): - """ - Copy the nrpe checks into place - - """ - NAGIOS_PLUGINS = '/usr/local/lib/nagios/plugins' - nrpe_files_dir = os.path.join(os.getenv('CHARM_DIR'), 'hooks', - 'charmhelpers', 'contrib', 'openstack', - 'files') - - if not os.path.exists(NAGIOS_PLUGINS): - os.makedirs(NAGIOS_PLUGINS) - for fname in glob.glob(os.path.join(nrpe_files_dir, "check_*")): - if os.path.isfile(fname): - shutil.copy2(fname, - os.path.join(NAGIOS_PLUGINS, os.path.basename(fname))) - - -def add_haproxy_checks(nrpe, unit_name): - """ - Add checks for each service in list - - :param NRPE nrpe: NRPE object to add check to - :param str unit_name: Unit name to use in check description - """ - nrpe.add_check( - shortname='haproxy_servers', - description='Check HAProxy {%s}' % unit_name, - check_cmd='check_haproxy.sh') - nrpe.add_check( - shortname='haproxy_queue', - description='Check HAProxy queue depth {%s}' % unit_name, - check_cmd='check_haproxy_queue_depth.sh') diff --git a/hooks/charmhelpers/contrib/charmsupport/volumes.py b/hooks/charmhelpers/contrib/charmsupport/volumes.py deleted file mode 100644 index 7ea43f0..0000000 --- a/hooks/charmhelpers/contrib/charmsupport/volumes.py +++ /dev/null @@ -1,173 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -''' -Functions for managing volumes in juju units. One volume is supported per unit. -Subordinates may have their own storage, provided it is on its own partition. - -Configuration stanzas:: - - volume-ephemeral: - type: boolean - default: true - description: > - If false, a volume is mounted as sepecified in "volume-map" - If true, ephemeral storage will be used, meaning that log data - will only exist as long as the machine. YOU HAVE BEEN WARNED. - volume-map: - type: string - default: {} - description: > - YAML map of units to device names, e.g: - "{ rsyslog/0: /dev/vdb, rsyslog/1: /dev/vdb }" - Service units will raise a configure-error if volume-ephemeral - is 'true' and no volume-map value is set. Use 'juju set' to set a - value and 'juju resolved' to complete configuration. - -Usage:: - - from charmsupport.volumes import configure_volume, VolumeConfigurationError - from charmsupport.hookenv import log, ERROR - def post_mount_hook(): - stop_service('myservice') - def post_mount_hook(): - start_service('myservice') - - if __name__ == '__main__': - try: - configure_volume(before_change=pre_mount_hook, - after_change=post_mount_hook) - except VolumeConfigurationError: - log('Storage could not be configured', ERROR) - -''' - -# XXX: Known limitations -# - fstab is neither consulted nor updated - -import os -from charmhelpers.core import hookenv -from charmhelpers.core import host -import yaml - - -MOUNT_BASE = '/srv/juju/volumes' - - -class VolumeConfigurationError(Exception): - '''Volume configuration data is missing or invalid''' - pass - - -def get_config(): - '''Gather and sanity-check volume configuration data''' - volume_config = {} - config = hookenv.config() - - errors = False - - if config.get('volume-ephemeral') in (True, 'True', 'true', 'Yes', 'yes'): - volume_config['ephemeral'] = True - else: - volume_config['ephemeral'] = False - - try: - volume_map = yaml.safe_load(config.get('volume-map', '{}')) - except yaml.YAMLError as e: - hookenv.log("Error parsing YAML volume-map: {}".format(e), - hookenv.ERROR) - errors = True - if volume_map is None: - # probably an empty string - volume_map = {} - elif not isinstance(volume_map, dict): - hookenv.log("Volume-map should be a dictionary, not {}".format( - type(volume_map))) - errors = True - - volume_config['device'] = volume_map.get(os.environ['JUJU_UNIT_NAME']) - if volume_config['device'] and volume_config['ephemeral']: - # asked for ephemeral storage but also defined a volume ID - hookenv.log('A volume is defined for this unit, but ephemeral ' - 'storage was requested', hookenv.ERROR) - errors = True - elif not volume_config['device'] and not volume_config['ephemeral']: - # asked for permanent storage but did not define volume ID - hookenv.log('Ephemeral storage was requested, but there is no volume ' - 'defined for this unit.', hookenv.ERROR) - errors = True - - unit_mount_name = hookenv.local_unit().replace('/', '-') - volume_config['mountpoint'] = os.path.join(MOUNT_BASE, unit_mount_name) - - if errors: - return None - return volume_config - - -def mount_volume(config): - if os.path.exists(config['mountpoint']): - if not os.path.isdir(config['mountpoint']): - hookenv.log('Not a directory: {}'.format(config['mountpoint'])) - raise VolumeConfigurationError() - else: - host.mkdir(config['mountpoint']) - if os.path.ismount(config['mountpoint']): - unmount_volume(config) - if not host.mount(config['device'], config['mountpoint'], persist=True): - raise VolumeConfigurationError() - - -def unmount_volume(config): - if os.path.ismount(config['mountpoint']): - if not host.umount(config['mountpoint'], persist=True): - raise VolumeConfigurationError() - - -def managed_mounts(): - '''List of all mounted managed volumes''' - return filter(lambda mount: mount[0].startswith(MOUNT_BASE), host.mounts()) - - -def configure_volume(before_change=lambda: None, after_change=lambda: None): - '''Set up storage (or don't) according to the charm's volume configuration. - Returns the mount point or "ephemeral". before_change and after_change - are optional functions to be called if the volume configuration changes. - ''' - - config = get_config() - if not config: - hookenv.log('Failed to read volume configuration', hookenv.CRITICAL) - raise VolumeConfigurationError() - - if config['ephemeral']: - if os.path.ismount(config['mountpoint']): - before_change() - unmount_volume(config) - after_change() - return 'ephemeral' - else: - # persistent storage - if os.path.ismount(config['mountpoint']): - mounts = dict(managed_mounts()) - if mounts.get(config['mountpoint']) != config['device']: - before_change() - unmount_volume(config) - mount_volume(config) - after_change() - else: - before_change() - mount_volume(config) - after_change() - return config['mountpoint'] diff --git a/hooks/charmhelpers/contrib/hardening/README.hardening.md b/hooks/charmhelpers/contrib/hardening/README.hardening.md deleted file mode 100644 index 91280c0..0000000 --- a/hooks/charmhelpers/contrib/hardening/README.hardening.md +++ /dev/null @@ -1,38 +0,0 @@ -# Juju charm-helpers hardening library - -## Description - -This library provides multiple implementations of system and application -hardening that conform to the standards of http://hardening.io/. - -Current implementations include: - - * OS - * SSH - * MySQL - * Apache - -## Requirements - -* Juju Charms - -## Usage - -1. Synchronise this library into your charm and add the harden() decorator - (from contrib.hardening.harden) to any functions or methods you want to use - to trigger hardening of your application/system. - -2. Add a config option called 'harden' to your charm config.yaml and set it to - a space-delimited list of hardening modules you want to run e.g. "os ssh" - -3. Override any config defaults (contrib.hardening.defaults) by adding a file - called hardening.yaml to your charm root containing the name(s) of the - modules whose settings you want override at root level and then any settings - with overrides e.g. - - os: - general: - desktop_enable: True - -4. Now just run your charm as usual and hardening will be applied each time the - hook runs. diff --git a/hooks/charmhelpers/contrib/hardening/__init__.py b/hooks/charmhelpers/contrib/hardening/__init__.py deleted file mode 100644 index 30a3e94..0000000 --- a/hooks/charmhelpers/contrib/hardening/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2016 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/hooks/charmhelpers/contrib/hardening/apache/__init__.py b/hooks/charmhelpers/contrib/hardening/apache/__init__.py deleted file mode 100644 index 58bebd8..0000000 --- a/hooks/charmhelpers/contrib/hardening/apache/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2016 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from os import path - -TEMPLATES_DIR = path.join(path.dirname(__file__), 'templates') diff --git a/hooks/charmhelpers/contrib/hardening/apache/checks/__init__.py b/hooks/charmhelpers/contrib/hardening/apache/checks/__init__.py deleted file mode 100644 index 3bc2ebd..0000000 --- a/hooks/charmhelpers/contrib/hardening/apache/checks/__init__.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2016 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from charmhelpers.core.hookenv import ( - log, - DEBUG, -) -from charmhelpers.contrib.hardening.apache.checks import config - - -def run_apache_checks(): - log("Starting Apache hardening checks.", level=DEBUG) - checks = config.get_audits() - for check in checks: - log("Running '%s' check" % (check.__class__.__name__), level=DEBUG) - check.ensure_compliance() - - log("Apache hardening checks complete.", level=DEBUG) diff --git a/hooks/charmhelpers/contrib/hardening/apache/checks/config.py b/hooks/charmhelpers/contrib/hardening/apache/checks/config.py deleted file mode 100644 index 06482aa..0000000 --- a/hooks/charmhelpers/contrib/hardening/apache/checks/config.py +++ /dev/null @@ -1,101 +0,0 @@ -# Copyright 2016 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import re -import subprocess - - -from charmhelpers.core.hookenv import ( - log, - INFO, -) -from charmhelpers.contrib.hardening.audits.file import ( - FilePermissionAudit, - DirectoryPermissionAudit, - NoReadWriteForOther, - TemplatedFile, - DeletedFile -) -from charmhelpers.contrib.hardening.audits.apache import DisabledModuleAudit -from charmhelpers.contrib.hardening.apache import TEMPLATES_DIR -from charmhelpers.contrib.hardening import utils - - -def get_audits(): - """Get Apache hardening config audits. - - :returns: dictionary of audits - """ - if subprocess.call(['which', 'apache2'], stdout=subprocess.PIPE) != 0: - log("Apache server does not appear to be installed on this node - " - "skipping apache hardening", level=INFO) - return [] - - context = ApacheConfContext() - settings = utils.get_settings('apache') - audits = [ - FilePermissionAudit(paths=os.path.join( - settings['common']['apache_dir'], 'apache2.conf'), - user='root', group='root', mode=0o0640), - - TemplatedFile(os.path.join(settings['common']['apache_dir'], - 'mods-available/alias.conf'), - context, - TEMPLATES_DIR, - mode=0o0640, - user='root', - service_actions=[{'service': 'apache2', - 'actions': ['restart']}]), - - TemplatedFile(os.path.join(settings['common']['apache_dir'], - 'conf-enabled/99-hardening.conf'), - context, - TEMPLATES_DIR, - mode=0o0640, - user='root', - service_actions=[{'service': 'apache2', - 'actions': ['restart']}]), - - DirectoryPermissionAudit(settings['common']['apache_dir'], - user='root', - group='root', - mode=0o0750), - - DisabledModuleAudit(settings['hardening']['modules_to_disable']), - - NoReadWriteForOther(settings['common']['apache_dir']), - - DeletedFile(['/var/www/html/index.html']) - ] - - return audits - - -class ApacheConfContext(object): - """Defines the set of key/value pairs to set in a apache config file. - - This context, when called, will return a dictionary containing the - key/value pairs of setting to specify in the - /etc/apache/conf-enabled/hardening.conf file. - """ - def __call__(self): - settings = utils.get_settings('apache') - ctxt = settings['hardening'] - - out = subprocess.check_output(['apache2', '-v']) - ctxt['apache_version'] = re.search(r'.+version: Apache/(.+?)\s.+', - out).group(1) - ctxt['apache_icondir'] = '/usr/share/apache2/icons/' - return ctxt diff --git a/hooks/charmhelpers/contrib/hardening/apache/templates/99-hardening.conf b/hooks/charmhelpers/contrib/hardening/apache/templates/99-hardening.conf deleted file mode 100644 index 22b6804..0000000 --- a/hooks/charmhelpers/contrib/hardening/apache/templates/99-hardening.conf +++ /dev/null @@ -1,32 +0,0 @@ -############################################################################### -# WARNING: This configuration file is maintained by Juju. Local changes may -# be overwritten. -############################################################################### - - - - # http://httpd.apache.org/docs/2.4/upgrading.html - {% if apache_version > '2.2' -%} - Require all granted - {% else -%} - Order Allow,Deny - Deny from all - {% endif %} - - - - - Options -Indexes -FollowSymLinks - AllowOverride None - - - - Options -Indexes -FollowSymLinks - AllowOverride None - - -TraceEnable {{ traceenable }} -ServerTokens {{ servertokens }} - -SSLHonorCipherOrder {{ honor_cipher_order }} -SSLCipherSuite {{ cipher_suite }} diff --git a/hooks/charmhelpers/contrib/hardening/apache/templates/__init__.py b/hooks/charmhelpers/contrib/hardening/apache/templates/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/hooks/charmhelpers/contrib/hardening/apache/templates/alias.conf b/hooks/charmhelpers/contrib/hardening/apache/templates/alias.conf deleted file mode 100644 index e46a58a..0000000 --- a/hooks/charmhelpers/contrib/hardening/apache/templates/alias.conf +++ /dev/null @@ -1,31 +0,0 @@ -############################################################################### -# WARNING: This configuration file is maintained by Juju. Local changes may -# be overwritten. -############################################################################### - - # - # Aliases: Add here as many aliases as you need (with no limit). The format is - # Alias fakename realname - # - # Note that if you include a trailing / on fakename then the server will - # require it to be present in the URL. So "/icons" isn't aliased in this - # example, only "/icons/". If the fakename is slash-terminated, then the - # realname must also be slash terminated, and if the fakename omits the - # trailing slash, the realname must also omit it. - # - # We include the /icons/ alias for FancyIndexed directory listings. If - # you do not use FancyIndexing, you may comment this out. - # - Alias /icons/ "{{ apache_icondir }}/" - - - Options -Indexes -MultiViews -FollowSymLinks - AllowOverride None -{% if apache_version == '2.4' -%} - Require all granted -{% else -%} - Order allow,deny - Allow from all -{% endif %} - - diff --git a/hooks/charmhelpers/contrib/hardening/audits/__init__.py b/hooks/charmhelpers/contrib/hardening/audits/__init__.py deleted file mode 100644 index 6dd5b05..0000000 --- a/hooks/charmhelpers/contrib/hardening/audits/__init__.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2016 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -class BaseAudit(object): # NO-QA - """Base class for hardening checks. - - The lifecycle of a hardening check is to first check to see if the system - is in compliance for the specified check. If it is not in compliance, the - check method will return a value which will be supplied to the. - """ - def __init__(self, *args, **kwargs): - self.unless = kwargs.get('unless', None) - super(BaseAudit, self).__init__() - - def ensure_compliance(self): - """Checks to see if the current hardening check is in compliance or - not. - - If the check that is performed is not in compliance, then an exception - should be raised. - """ - pass - - def _take_action(self): - """Determines whether to perform the action or not. - - Checks whether or not an action should be taken. This is determined by - the truthy value for the unless parameter. If unless is a callback - method, it will be invoked with no parameters in order to determine - whether or not the action should be taken. Otherwise, the truthy value - of the unless attribute will determine if the action should be - performed. - """ - # Do the action if there isn't an unless override. - if self.unless is None: - return True - - # Invoke the callback if there is one. - if hasattr(self.unless, '__call__'): - return not self.unless() - - return not self.unless diff --git a/hooks/charmhelpers/contrib/hardening/audits/apache.py b/hooks/charmhelpers/contrib/hardening/audits/apache.py deleted file mode 100644 index d32bf44..0000000 --- a/hooks/charmhelpers/contrib/hardening/audits/apache.py +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright 2016 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -import subprocess - -from six import string_types - -from charmhelpers.core.hookenv import ( - log, - INFO, - ERROR, -) - -from charmhelpers.contrib.hardening.audits import BaseAudit - - -class DisabledModuleAudit(BaseAudit): - """Audits Apache2 modules. - - Determines if the apache2 modules are enabled. If the modules are enabled - then they are removed in the ensure_compliance. - """ - def __init__(self, modules): - if modules is None: - self.modules = [] - elif isinstance(modules, string_types): - self.modules = [modules] - else: - self.modules = modules - - def ensure_compliance(self): - """Ensures that the modules are not loaded.""" - if not self.modules: - return - - try: - loaded_modules = self._get_loaded_modules() - non_compliant_modules = [] - for module in self.modules: - if module in loaded_modules: - log("Module '%s' is enabled but should not be." % - (module), level=INFO) - non_compliant_modules.append(module) - - if len(non_compliant_modules) == 0: - return - - for module in non_compliant_modules: - self._disable_module(module) - self._restart_apache() - except subprocess.CalledProcessError as e: - log('Error occurred auditing apache module compliance. ' - 'This may have been already reported. ' - 'Output is: %s' % e.output, level=ERROR) - - @staticmethod - def _get_loaded_modules(): - """Returns the modules which are enabled in Apache.""" - output = subprocess.check_output(['apache2ctl', '-M']) - modules = [] - for line in output.splitlines(): - # Each line of the enabled module output looks like: - # module_name (static|shared) - # Plus a header line at the top of the output which is stripped - # out by the regex. - matcher = re.search(r'^ (\S*)_module (\S*)', line) - if matcher: - modules.append(matcher.group(1)) - return modules - - @staticmethod - def _disable_module(module): - """Disables the specified module in Apache.""" - try: - subprocess.check_call(['a2dismod', module]) - except subprocess.CalledProcessError as e: - # Note: catch error here to allow the attempt of disabling - # multiple modules in one go rather than failing after the - # first module fails. - log('Error occurred disabling module %s. ' - 'Output is: %s' % (module, e.output), level=ERROR) - - @staticmethod - def _restart_apache(): - """Restarts the apache process""" - subprocess.check_output(['service', 'apache2', 'restart']) diff --git a/hooks/charmhelpers/contrib/hardening/audits/apt.py b/hooks/charmhelpers/contrib/hardening/audits/apt.py deleted file mode 100644 index 3dc14e3..0000000 --- a/hooks/charmhelpers/contrib/hardening/audits/apt.py +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright 2016 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import # required for external apt import -from apt import apt_pkg -from six import string_types - -from charmhelpers.fetch import ( - apt_cache, - apt_purge -) -from charmhelpers.core.hookenv import ( - log, - DEBUG, - WARNING, -) -from charmhelpers.contrib.hardening.audits import BaseAudit - - -class AptConfig(BaseAudit): - - def __init__(self, config, **kwargs): - self.config = config - - def verify_config(self): - apt_pkg.init() - for cfg in self.config: - value = apt_pkg.config.get(cfg['key'], cfg.get('default', '')) - if value and value != cfg['expected']: - log("APT config '%s' has unexpected value '%s' " - "(expected='%s')" % - (cfg['key'], value, cfg['expected']), level=WARNING) - - def ensure_compliance(self): - self.verify_config() - - -class RestrictedPackages(BaseAudit): - """Class used to audit restricted packages on the system.""" - - def __init__(self, pkgs, **kwargs): - super(RestrictedPackages, self).__init__(**kwargs) - if isinstance(pkgs, string_types) or not hasattr(pkgs, '__iter__'): - self.pkgs = [pkgs] - else: - self.pkgs = pkgs - - def ensure_compliance(self): - cache = apt_cache() - - for p in self.pkgs: - if p not in cache: - continue - - pkg = cache[p] - if not self.is_virtual_package(pkg): - if not pkg.current_ver: - log("Package '%s' is not installed." % pkg.name, - level=DEBUG) - continue - else: - log("Restricted package '%s' is installed" % pkg.name, - level=WARNING) - self.delete_package(cache, pkg) - else: - log("Checking restricted virtual package '%s' provides" % - pkg.name, level=DEBUG) - self.delete_package(cache, pkg) - - def delete_package(self, cache, pkg): - """Deletes the package from the system. - - Deletes the package form the system, properly handling virtual - packages. - - :param cache: the apt cache - :param pkg: the package to remove - """ - if self.is_virtual_package(pkg): - log("Package '%s' appears to be virtual - purging provides" % - pkg.name, level=DEBUG) - for _p in pkg.provides_list: - self.delete_package(cache, _p[2].parent_pkg) - elif not pkg.current_ver: - log("Package '%s' not installed" % pkg.name, level=DEBUG) - return - else: - log("Purging package '%s'" % pkg.name, level=DEBUG) - apt_purge(pkg.name) - - def is_virtual_package(self, pkg): - return pkg.has_provides and not pkg.has_versions diff --git a/hooks/charmhelpers/contrib/hardening/audits/file.py b/hooks/charmhelpers/contrib/hardening/audits/file.py deleted file mode 100644 index 257c635..0000000 --- a/hooks/charmhelpers/contrib/hardening/audits/file.py +++ /dev/null @@ -1,550 +0,0 @@ -# Copyright 2016 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import grp -import os -import pwd -import re - -from subprocess import ( - CalledProcessError, - check_output, - check_call, -) -from traceback import format_exc -from six import string_types -from stat import ( - S_ISGID, - S_ISUID -) - -from charmhelpers.core.hookenv import ( - log, - DEBUG, - INFO, - WARNING, - ERROR, -) -from charmhelpers.core import unitdata -from charmhelpers.core.host import file_hash -from charmhelpers.contrib.hardening.audits import BaseAudit -from charmhelpers.contrib.hardening.templating import ( - get_template_path, - render_and_write, -) -from charmhelpers.contrib.hardening import utils - - -class BaseFileAudit(BaseAudit): - """Base class for file audits. - - Provides api stubs for compliance check flow that must be used by any class - that implemented this one. - """ - - def __init__(self, paths, always_comply=False, *args, **kwargs): - """ - :param paths: string path of list of paths of files we want to apply - compliance checks are criteria to. - :param always_comply: if true compliance criteria is always applied - else compliance is skipped for non-existent - paths. - """ - super(BaseFileAudit, self).__init__(*args, **kwargs) - self.always_comply = always_comply - if isinstance(paths, string_types) or not hasattr(paths, '__iter__'): - self.paths = [paths] - else: - self.paths = paths - - def ensure_compliance(self): - """Ensure that the all registered files comply to registered criteria. - """ - for p in self.paths: - if os.path.exists(p): - if self.is_compliant(p): - continue - - log('File %s is not in compliance.' % p, level=INFO) - else: - if not self.always_comply: - log("Non-existent path '%s' - skipping compliance check" - % (p), level=INFO) - continue - - if self._take_action(): - log("Applying compliance criteria to '%s'" % (p), level=INFO) - self.comply(p) - - def is_compliant(self, path): - """Audits the path to see if it is compliance. - - :param path: the path to the file that should be checked. - """ - raise NotImplementedError - - def comply(self, path): - """Enforces the compliance of a path. - - :param path: the path to the file that should be enforced. - """ - raise NotImplementedError - - @classmethod - def _get_stat(cls, path): - """Returns the Posix st_stat information for the specified file path. - - :param path: the path to get the st_stat information for. - :returns: an st_stat object for the path or None if the path doesn't - exist. - """ - return os.stat(path) - - -class FilePermissionAudit(BaseFileAudit): - """Implements an audit for file permissions and ownership for a user. - - This class implements functionality that ensures that a specific user/group - will own the file(s) specified and that the permissions specified are - applied properly to the file. - """ - def __init__(self, paths, user, group=None, mode=0o600, **kwargs): - self.user = user - self.group = group - self.mode = mode - super(FilePermissionAudit, self).__init__(paths, user, group, mode, - **kwargs) - - @property - def user(self): - return self._user - - @user.setter - def user(self, name): - try: - user = pwd.getpwnam(name) - except KeyError: - log('Unknown user %s' % name, level=ERROR) - user = None - self._user = user - - @property - def group(self): - return self._group - - @group.setter - def group(self, name): - try: - group = None - if name: - group = grp.getgrnam(name) - else: - group = grp.getgrgid(self.user.pw_gid) - except KeyError: - log('Unknown group %s' % name, level=ERROR) - self._group = group - - def is_compliant(self, path): - """Checks if the path is in compliance. - - Used to determine if the path specified meets the necessary - requirements to be in compliance with the check itself. - - :param path: the file path to check - :returns: True if the path is compliant, False otherwise. - """ - stat = self._get_stat(path) - user = self.user - group = self.group - - compliant = True - if stat.st_uid != user.pw_uid or stat.st_gid != group.gr_gid: - log('File %s is not owned by %s:%s.' % (path, user.pw_name, - group.gr_name), - level=INFO) - compliant = False - - # POSIX refers to the st_mode bits as corresponding to both the - # file type and file permission bits, where the least significant 12 - # bits (o7777) are the suid (11), sgid (10), sticky bits (9), and the - # file permission bits (8-0) - perms = stat.st_mode & 0o7777 - if perms != self.mode: - log('File %s has incorrect permissions, currently set to %s' % - (path, oct(stat.st_mode & 0o7777)), level=INFO) - compliant = False - - return compliant - - def comply(self, path): - """Issues a chown and chmod to the file paths specified.""" - utils.ensure_permissions(path, self.user.pw_name, self.group.gr_name, - self.mode) - - -class DirectoryPermissionAudit(FilePermissionAudit): - """Performs a permission check for the specified directory path.""" - - def __init__(self, paths, user, group=None, mode=0o600, - recursive=True, **kwargs): - super(DirectoryPermissionAudit, self).__init__(paths, user, group, - mode, **kwargs) - self.recursive = recursive - - def is_compliant(self, path): - """Checks if the directory is compliant. - - Used to determine if the path specified and all of its children - directories are in compliance with the check itself. - - :param path: the directory path to check - :returns: True if the directory tree is compliant, otherwise False. - """ - if not os.path.isdir(path): - log('Path specified %s is not a directory.' % path, level=ERROR) - raise ValueError("%s is not a directory." % path) - - if not self.recursive: - return super(DirectoryPermissionAudit, self).is_compliant(path) - - compliant = True - for root, dirs, _ in os.walk(path): - if len(dirs) > 0: - continue - - if not super(DirectoryPermissionAudit, self).is_compliant(root): - compliant = False - continue - - return compliant - - def comply(self, path): - for root, dirs, _ in os.walk(path): - if len(dirs) > 0: - super(DirectoryPermissionAudit, self).comply(root) - - -class ReadOnly(BaseFileAudit): - """Audits that files and folders are read only.""" - def __init__(self, paths, *args, **kwargs): - super(ReadOnly, self).__init__(paths=paths, *args, **kwargs) - - def is_compliant(self, path): - try: - output = check_output(['find', path, '-perm', '-go+w', - '-type', 'f']).strip() - - # The find above will find any files which have permission sets - # which allow too broad of write access. As such, the path is - # compliant if there is no output. - if output: - return False - - return True - except CalledProcessError as e: - log('Error occurred checking finding writable files for %s. ' - 'Error information is: command %s failed with returncode ' - '%d and output %s.\n%s' % (path, e.cmd, e.returncode, e.output, - format_exc(e)), level=ERROR) - return False - - def comply(self, path): - try: - check_output(['chmod', 'go-w', '-R', path]) - except CalledProcessError as e: - log('Error occurred removing writeable permissions for %s. ' - 'Error information is: command %s failed with returncode ' - '%d and output %s.\n%s' % (path, e.cmd, e.returncode, e.output, - format_exc(e)), level=ERROR) - - -class NoReadWriteForOther(BaseFileAudit): - """Ensures that the files found under the base path are readable or - writable by anyone other than the owner or the group. - """ - def __init__(self, paths): - super(NoReadWriteForOther, self).__init__(paths) - - def is_compliant(self, path): - try: - cmd = ['find', path, '-perm', '-o+r', '-type', 'f', '-o', - '-perm', '-o+w', '-type', 'f'] - output = check_output(cmd).strip() - - # The find above here will find any files which have read or - # write permissions for other, meaning there is too broad of access - # to read/write the file. As such, the path is compliant if there's - # no output. - if output: - return False - - return True - except CalledProcessError as e: - log('Error occurred while finding files which are readable or ' - 'writable to the world in %s. ' - 'Command output is: %s.' % (path, e.output), level=ERROR) - - def comply(self, path): - try: - check_output(['chmod', '-R', 'o-rw', path]) - except CalledProcessError as e: - log('Error occurred attempting to change modes of files under ' - 'path %s. Output of command is: %s' % (path, e.output)) - - -class NoSUIDSGIDAudit(BaseFileAudit): - """Audits that specified files do not have SUID/SGID bits set.""" - def __init__(self, paths, *args, **kwargs): - super(NoSUIDSGIDAudit, self).__init__(paths=paths, *args, **kwargs) - - def is_compliant(self, path): - stat = self._get_stat(path) - if (stat.st_mode & (S_ISGID | S_ISUID)) != 0: - return False - - return True - - def comply(self, path): - try: - log('Removing suid/sgid from %s.' % path, level=DEBUG) - check_output(['chmod', '-s', path]) - except CalledProcessError as e: - log('Error occurred removing suid/sgid from %s.' - 'Error information is: command %s failed with returncode ' - '%d and output %s.\n%s' % (path, e.cmd, e.returncode, e.output, - format_exc(e)), level=ERROR) - - -class TemplatedFile(BaseFileAudit): - """The TemplatedFileAudit audits the contents of a templated file. - - This audit renders a file from a template, sets the appropriate file - permissions, then generates a hashsum with which to check the content - changed. - """ - def __init__(self, path, context, template_dir, mode, user='root', - group='root', service_actions=None, **kwargs): - self.context = context - self.user = user - self.group = group - self.mode = mode - self.template_dir = template_dir - self.service_actions = service_actions - super(TemplatedFile, self).__init__(paths=path, always_comply=True, - **kwargs) - - def is_compliant(self, path): - """Determines if the templated file is compliant. - - A templated file is only compliant if it has not changed (as - determined by its sha256 hashsum) AND its file permissions are set - appropriately. - - :param path: the path to check compliance. - """ - same_templates = self.templates_match(path) - same_content = self.contents_match(path) - same_permissions = self.permissions_match(path) - - if same_content and same_permissions and same_templates: - return True - - return False - - def run_service_actions(self): - """Run any actions on services requested.""" - if not self.service_actions: - return - - for svc_action in self.service_actions: - name = svc_action['service'] - actions = svc_action['actions'] - log("Running service '%s' actions '%s'" % (name, actions), - level=DEBUG) - for action in actions: - cmd = ['service', name, action] - try: - check_call(cmd) - except CalledProcessError as exc: - log("Service name='%s' action='%s' failed - %s" % - (name, action, exc), level=WARNING) - - def comply(self, path): - """Ensures the contents and the permissions of the file. - - :param path: the path to correct - """ - dirname = os.path.dirname(path) - if not os.path.exists(dirname): - os.makedirs(dirname) - - self.pre_write() - render_and_write(self.template_dir, path, self.context()) - utils.ensure_permissions(path, self.user, self.group, self.mode) - self.run_service_actions() - self.save_checksum(path) - self.post_write() - - def pre_write(self): - """Invoked prior to writing the template.""" - pass - - def post_write(self): - """Invoked after writing the template.""" - pass - - def templates_match(self, path): - """Determines if the template files are the same. - - The template file equality is determined by the hashsum of the - template files themselves. If there is no hashsum, then the content - cannot be sure to be the same so treat it as if they changed. - Otherwise, return whether or not the hashsums are the same. - - :param path: the path to check - :returns: boolean - """ - template_path = get_template_path(self.template_dir, path) - key = 'hardening:template:%s' % template_path - template_checksum = file_hash(template_path) - kv = unitdata.kv() - stored_tmplt_checksum = kv.get(key) - if not stored_tmplt_checksum: - kv.set(key, template_checksum) - kv.flush() - log('Saved template checksum for %s.' % template_path, - level=DEBUG) - # Since we don't have a template checksum, then assume it doesn't - # match and return that the template is different. - return False - elif stored_tmplt_checksum != template_checksum: - kv.set(key, template_checksum) - kv.flush() - log('Updated template checksum for %s.' % template_path, - level=DEBUG) - return False - - # Here the template hasn't changed based upon the calculated - # checksum of the template and what was previously stored. - return True - - def contents_match(self, path): - """Determines if the file content is the same. - - This is determined by comparing hashsum of the file contents and - the saved hashsum. If there is no hashsum, then the content cannot - be sure to be the same so treat them as if they are not the same. - Otherwise, return True if the hashsums are the same, False if they - are not the same. - - :param path: the file to check. - """ - checksum = file_hash(path) - - kv = unitdata.kv() - stored_checksum = kv.get('hardening:%s' % path) - if not stored_checksum: - # If the checksum hasn't been generated, return False to ensure - # the file is written and the checksum stored. - log('Checksum for %s has not been calculated.' % path, level=DEBUG) - return False - elif stored_checksum != checksum: - log('Checksum mismatch for %s.' % path, level=DEBUG) - return False - - return True - - def permissions_match(self, path): - """Determines if the file owner and permissions match. - - :param path: the path to check. - """ - audit = FilePermissionAudit(path, self.user, self.group, self.mode) - return audit.is_compliant(path) - - def save_checksum(self, path): - """Calculates and saves the checksum for the path specified. - - :param path: the path of the file to save the checksum. - """ - checksum = file_hash(path) - kv = unitdata.kv() - kv.set('hardening:%s' % path, checksum) - kv.flush() - - -class DeletedFile(BaseFileAudit): - """Audit to ensure that a file is deleted.""" - def __init__(self, paths): - super(DeletedFile, self).__init__(paths) - - def is_compliant(self, path): - return not os.path.exists(path) - - def comply(self, path): - os.remove(path) - - -class FileContentAudit(BaseFileAudit): - """Audit the contents of a file.""" - def __init__(self, paths, cases, **kwargs): - # Cases we expect to pass - self.pass_cases = cases.get('pass', []) - # Cases we expect to fail - self.fail_cases = cases.get('fail', []) - super(FileContentAudit, self).__init__(paths, **kwargs) - - def is_compliant(self, path): - """ - Given a set of content matching cases i.e. tuple(regex, bool) where - bool value denotes whether or not regex is expected to match, check that - all cases match as expected with the contents of the file. Cases can be - expected to pass of fail. - - :param path: Path of file to check. - :returns: Boolean value representing whether or not all cases are - found to be compliant. - """ - log("Auditing contents of file '%s'" % (path), level=DEBUG) - with open(path, 'r') as fd: - contents = fd.read() - - matches = 0 - for pattern in self.pass_cases: - key = re.compile(pattern, flags=re.MULTILINE) - results = re.search(key, contents) - if results: - matches += 1 - else: - log("Pattern '%s' was expected to pass but instead it failed" - % (pattern), level=WARNING) - - for pattern in self.fail_cases: - key = re.compile(pattern, flags=re.MULTILINE) - results = re.search(key, contents) - if not results: - matches += 1 - else: - log("Pattern '%s' was expected to fail but instead it passed" - % (pattern), level=WARNING) - - total = len(self.pass_cases) + len(self.fail_cases) - log("Checked %s cases and %s passed" % (total, matches), level=DEBUG) - return matches == total - - def comply(self, *args, **kwargs): - """NOOP since we just issue warnings. This is to avoid the - NotImplememtedError. - """ - log("Not applying any compliance criteria, only checks.", level=INFO) diff --git a/hooks/charmhelpers/contrib/hardening/defaults/__init__.py b/hooks/charmhelpers/contrib/hardening/defaults/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/hooks/charmhelpers/contrib/hardening/defaults/apache.yaml b/hooks/charmhelpers/contrib/hardening/defaults/apache.yaml deleted file mode 100644 index 0f940d4..0000000 --- a/hooks/charmhelpers/contrib/hardening/defaults/apache.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# NOTE: this file contains the default configuration for the 'apache' hardening -# code. If you want to override any settings you must add them to a file -# called hardening.yaml in the root directory of your charm using the -# name 'apache' as the root key followed by any of the following with new -# values. - -common: - apache_dir: '/etc/apache2' - -hardening: - traceenable: 'off' - allowed_http_methods: "GET POST" - modules_to_disable: [ cgi, cgid ] - servertokens: 'Prod' - honor_cipher_order: 'on' - cipher_suite: 'ALL:+MEDIUM:+HIGH:!LOW:!MD5:!RC4:!eNULL:!aNULL:!3DES' diff --git a/hooks/charmhelpers/contrib/hardening/defaults/apache.yaml.schema b/hooks/charmhelpers/contrib/hardening/defaults/apache.yaml.schema deleted file mode 100644 index c112137..0000000 --- a/hooks/charmhelpers/contrib/hardening/defaults/apache.yaml.schema +++ /dev/null @@ -1,12 +0,0 @@ -# NOTE: this schema must contain all valid keys from it's associated defaults -# file. It is used to validate user-provided overrides. -common: - apache_dir: - traceenable: - -hardening: - allowed_http_methods: - modules_to_disable: - servertokens: - honor_cipher_order: - cipher_suite: diff --git a/hooks/charmhelpers/contrib/hardening/defaults/mysql.yaml b/hooks/charmhelpers/contrib/hardening/defaults/mysql.yaml deleted file mode 100644 index 682d22b..0000000 --- a/hooks/charmhelpers/contrib/hardening/defaults/mysql.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# NOTE: this file contains the default configuration for the 'mysql' hardening -# code. If you want to override any settings you must add them to a file -# called hardening.yaml in the root directory of your charm using the -# name 'mysql' as the root key followed by any of the following with new -# values. - -hardening: - mysql-conf: /etc/mysql/my.cnf - hardening-conf: /etc/mysql/conf.d/hardening.cnf - -security: - # @see http://www.symantec.com/connect/articles/securing-mysql-step-step - # @see http://dev.mysql.com/doc/refman/5.7/en/server-options.html#option_mysqld_chroot - chroot: None - - # @see http://dev.mysql.com/doc/refman/5.7/en/server-options.html#option_mysqld_safe-user-create - safe-user-create: 1 - - # @see http://dev.mysql.com/doc/refman/5.7/en/server-options.html#option_mysqld_secure-auth - secure-auth: 1 - - # @see http://dev.mysql.com/doc/refman/5.7/en/server-options.html#option_mysqld_symbolic-links - skip-symbolic-links: 1 - - # @see http://dev.mysql.com/doc/refman/5.7/en/server-options.html#option_mysqld_skip-show-database - skip-show-database: True - - # @see http://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_local_infile - local-infile: 0 - - # @see https://dev.mysql.com/doc/refman/5.7/en/server-options.html#option_mysqld_allow-suspicious-udfs - allow-suspicious-udfs: 0 - - # @see https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_automatic_sp_privileges - automatic-sp-privileges: 0 - - # @see https://dev.mysql.com/doc/refman/5.7/en/server-options.html#option_mysqld_secure-file-priv - secure-file-priv: /tmp diff --git a/hooks/charmhelpers/contrib/hardening/defaults/mysql.yaml.schema b/hooks/charmhelpers/contrib/hardening/defaults/mysql.yaml.schema deleted file mode 100644 index 2edf325..0000000 --- a/hooks/charmhelpers/contrib/hardening/defaults/mysql.yaml.schema +++ /dev/null @@ -1,15 +0,0 @@ -# NOTE: this schema must contain all valid keys from it's associated defaults -# file. It is used to validate user-provided overrides. -hardening: - mysql-conf: - hardening-conf: -security: - chroot: - safe-user-create: - secure-auth: - skip-symbolic-links: - skip-show-database: - local-infile: - allow-suspicious-udfs: - automatic-sp-privileges: - secure-file-priv: diff --git a/hooks/charmhelpers/contrib/hardening/defaults/os.yaml b/hooks/charmhelpers/contrib/hardening/defaults/os.yaml deleted file mode 100644 index 9a8627b..0000000 --- a/hooks/charmhelpers/contrib/hardening/defaults/os.yaml +++ /dev/null @@ -1,68 +0,0 @@ -# NOTE: this file contains the default configuration for the 'os' hardening -# code. If you want to override any settings you must add them to a file -# called hardening.yaml in the root directory of your charm using the -# name 'os' as the root key followed by any of the following with new -# values. - -general: - desktop_enable: False # (type:boolean) - -environment: - extra_user_paths: [] - umask: 027 - root_path: / - -auth: - pw_max_age: 60 - # discourage password cycling - pw_min_age: 7 - retries: 5 - lockout_time: 600 - timeout: 60 - allow_homeless: False # (type:boolean) - pam_passwdqc_enable: True # (type:boolean) - pam_passwdqc_options: 'min=disabled,disabled,16,12,8' - root_ttys: - console - tty1 - tty2 - tty3 - tty4 - tty5 - tty6 - uid_min: 1000 - gid_min: 1000 - sys_uid_min: 100 - sys_uid_max: 999 - sys_gid_min: 100 - sys_gid_max: 999 - chfn_restrict: - -security: - users_allow: [] - suid_sgid_enforce: True # (type:boolean) - # user-defined blacklist and whitelist - suid_sgid_blacklist: [] - suid_sgid_whitelist: [] - # if this is True, remove any suid/sgid bits from files that were not in the whitelist - suid_sgid_dry_run_on_unknown: False # (type:boolean) - suid_sgid_remove_from_unknown: False # (type:boolean) - # remove packages with known issues - packages_clean: True # (type:boolean) - packages_list: - xinetd - inetd - ypserv - telnet-server - rsh-server - rsync - kernel_enable_module_loading: True # (type:boolean) - kernel_enable_core_dump: False # (type:boolean) - ssh_tmout: 300 - -sysctl: - kernel_secure_sysrq: 244 # 4 + 16 + 32 + 64 + 128 - kernel_enable_sysrq: False # (type:boolean) - forwarding: False # (type:boolean) - ipv6_enable: False # (type:boolean) - arp_restricted: True # (type:boolean) diff --git a/hooks/charmhelpers/contrib/hardening/defaults/os.yaml.schema b/hooks/charmhelpers/contrib/hardening/defaults/os.yaml.schema deleted file mode 100644 index cc3b9c2..0000000 --- a/hooks/charmhelpers/contrib/hardening/defaults/os.yaml.schema +++ /dev/null @@ -1,43 +0,0 @@ -# NOTE: this schema must contain all valid keys from it's associated defaults -# file. It is used to validate user-provided overrides. -general: - desktop_enable: -environment: - extra_user_paths: - umask: - root_path: -auth: - pw_max_age: - pw_min_age: - retries: - lockout_time: - timeout: - allow_homeless: - pam_passwdqc_enable: - pam_passwdqc_options: - root_ttys: - uid_min: - gid_min: - sys_uid_min: - sys_uid_max: - sys_gid_min: - sys_gid_max: - chfn_restrict: -security: - users_allow: - suid_sgid_enforce: - suid_sgid_blacklist: - suid_sgid_whitelist: - suid_sgid_dry_run_on_unknown: - suid_sgid_remove_from_unknown: - packages_clean: - packages_list: - kernel_enable_module_loading: - kernel_enable_core_dump: - ssh_tmout: -sysctl: - kernel_secure_sysrq: - kernel_enable_sysrq: - forwarding: - ipv6_enable: - arp_restricted: diff --git a/hooks/charmhelpers/contrib/hardening/defaults/ssh.yaml b/hooks/charmhelpers/contrib/hardening/defaults/ssh.yaml deleted file mode 100644 index cd529bc..0000000 --- a/hooks/charmhelpers/contrib/hardening/defaults/ssh.yaml +++ /dev/null @@ -1,49 +0,0 @@ -# NOTE: this file contains the default configuration for the 'ssh' hardening -# code. If you want to override any settings you must add them to a file -# called hardening.yaml in the root directory of your charm using the -# name 'ssh' as the root key followed by any of the following with new -# values. - -common: - service_name: 'ssh' - network_ipv6_enable: False # (type:boolean) - ports: [22] - remote_hosts: [] - -client: - package: 'openssh-client' - cbc_required: False # (type:boolean) - weak_hmac: False # (type:boolean) - weak_kex: False # (type:boolean) - roaming: False - password_authentication: 'no' - -server: - host_key_files: ['/etc/ssh/ssh_host_rsa_key', '/etc/ssh/ssh_host_dsa_key', - '/etc/ssh/ssh_host_ecdsa_key'] - cbc_required: False # (type:boolean) - weak_hmac: False # (type:boolean) - weak_kex: False # (type:boolean) - allow_root_with_key: False # (type:boolean) - allow_tcp_forwarding: 'no' - allow_agent_forwarding: 'no' - allow_x11_forwarding: 'no' - use_privilege_separation: 'sandbox' - listen_to: ['0.0.0.0'] - use_pam: 'no' - package: 'openssh-server' - password_authentication: 'no' - alive_interval: '600' - alive_count: '3' - sftp_enable: False # (type:boolean) - sftp_group: 'sftponly' - sftp_chroot: '/home/%u' - deny_users: [] - allow_users: [] - deny_groups: [] - allow_groups: [] - print_motd: 'no' - print_last_log: 'no' - use_dns: 'no' - max_auth_tries: 2 - max_sessions: 10 diff --git a/hooks/charmhelpers/contrib/hardening/defaults/ssh.yaml.schema b/hooks/charmhelpers/contrib/hardening/defaults/ssh.yaml.schema deleted file mode 100644 index d05e054..0000000 --- a/hooks/charmhelpers/contrib/hardening/defaults/ssh.yaml.schema +++ /dev/null @@ -1,42 +0,0 @@ -# NOTE: this schema must contain all valid keys from it's associated defaults -# file. It is used to validate user-provided overrides. -common: - service_name: - network_ipv6_enable: - ports: - remote_hosts: -client: - package: - cbc_required: - weak_hmac: - weak_kex: - roaming: - password_authentication: -server: - host_key_files: - cbc_required: - weak_hmac: - weak_kex: - allow_root_with_key: - allow_tcp_forwarding: - allow_agent_forwarding: - allow_x11_forwarding: - use_privilege_separation: - listen_to: - use_pam: - package: - password_authentication: - alive_interval: - alive_count: - sftp_enable: - sftp_group: - sftp_chroot: - deny_users: - allow_users: - deny_groups: - allow_groups: - print_motd: - print_last_log: - use_dns: - max_auth_tries: - max_sessions: diff --git a/hooks/charmhelpers/contrib/hardening/harden.py b/hooks/charmhelpers/contrib/hardening/harden.py deleted file mode 100644 index b55764c..0000000 --- a/hooks/charmhelpers/contrib/hardening/harden.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright 2016 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import six - -from collections import OrderedDict - -from charmhelpers.core.hookenv import ( - config, - log, - DEBUG, - WARNING, -) -from charmhelpers.contrib.hardening.host.checks import run_os_checks -from charmhelpers.contrib.hardening.ssh.checks import run_ssh_checks -from charmhelpers.contrib.hardening.mysql.checks import run_mysql_checks -from charmhelpers.contrib.hardening.apache.checks import run_apache_checks - - -def harden(overrides=None): - """Hardening decorator. - - This is the main entry point for running the hardening stack. In order to - run modules of the stack you must add this decorator to charm hook(s) and - ensure that your charm config.yaml contains the 'harden' option set to - one or more of the supported modules. Setting these will cause the - corresponding hardening code to be run when the hook fires. - - This decorator can and should be applied to more than one hook or function - such that hardening modules are called multiple times. This is because - subsequent calls will perform auditing checks that will report any changes - to resources hardened by the first run (and possibly perform compliance - actions as a result of any detected infractions). - - :param overrides: Optional list of stack modules used to override those - provided with 'harden' config. - :returns: Returns value returned by decorated function once executed. - """ - def _harden_inner1(f): - log("Hardening function '%s'" % (f.__name__), level=DEBUG) - - def _harden_inner2(*args, **kwargs): - RUN_CATALOG = OrderedDict([('os', run_os_checks), - ('ssh', run_ssh_checks), - ('mysql', run_mysql_checks), - ('apache', run_apache_checks)]) - - enabled = overrides or (config("harden") or "").split() - if enabled: - modules_to_run = [] - # modules will always be performed in the following order - for module, func in six.iteritems(RUN_CATALOG): - if module in enabled: - enabled.remove(module) - modules_to_run.append(func) - - if enabled: - log("Unknown hardening modules '%s' - ignoring" % - (', '.join(enabled)), level=WARNING) - - for hardener in modules_to_run: - log("Executing hardening module '%s'" % - (hardener.__name__), level=DEBUG) - hardener() - else: - log("No hardening applied to '%s'" % (f.__name__), level=DEBUG) - - return f(*args, **kwargs) - return _harden_inner2 - - return _harden_inner1 diff --git a/hooks/charmhelpers/contrib/hardening/host/__init__.py b/hooks/charmhelpers/contrib/hardening/host/__init__.py deleted file mode 100644 index 58bebd8..0000000 --- a/hooks/charmhelpers/contrib/hardening/host/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2016 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from os import path - -TEMPLATES_DIR = path.join(path.dirname(__file__), 'templates') diff --git a/hooks/charmhelpers/contrib/hardening/host/checks/__init__.py b/hooks/charmhelpers/contrib/hardening/host/checks/__init__.py deleted file mode 100644 index 0e7e409..0000000 --- a/hooks/charmhelpers/contrib/hardening/host/checks/__init__.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2016 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from charmhelpers.core.hookenv import ( - log, - DEBUG, -) -from charmhelpers.contrib.hardening.host.checks import ( - apt, - limits, - login, - minimize_access, - pam, - profile, - securetty, - suid_sgid, - sysctl -) - - -def run_os_checks(): - log("Starting OS hardening checks.", level=DEBUG) - checks = apt.get_audits() - checks.extend(limits.get_audits()) - checks.extend(login.get_audits()) - checks.extend(minimize_access.get_audits()) - checks.extend(pam.get_audits()) - checks.extend(profile.get_audits()) - checks.extend(securetty.get_audits()) - checks.extend(suid_sgid.get_audits()) - checks.extend(sysctl.get_audits()) - - for check in checks: - log("Running '%s' check" % (check.__class__.__name__), level=DEBUG) - check.ensure_compliance() - - log("OS hardening checks complete.", level=DEBUG) diff --git a/hooks/charmhelpers/contrib/hardening/host/checks/apt.py b/hooks/charmhelpers/contrib/hardening/host/checks/apt.py deleted file mode 100644 index 7ce41b0..0000000 --- a/hooks/charmhelpers/contrib/hardening/host/checks/apt.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2016 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from charmhelpers.contrib.hardening.utils import get_settings -from charmhelpers.contrib.hardening.audits.apt import ( - AptConfig, - RestrictedPackages, -) - - -def get_audits(): - """Get OS hardening apt audits. - - :returns: dictionary of audits - """ - audits = [AptConfig([{'key': 'APT::Get::AllowUnauthenticated', - 'expected': 'false'}])] - - settings = get_settings('os') - clean_packages = settings['security']['packages_clean'] - if clean_packages: - security_packages = settings['security']['packages_list'] - if security_packages: - audits.append(RestrictedPackages(security_packages)) - - return audits diff --git a/hooks/charmhelpers/contrib/hardening/host/checks/limits.py b/hooks/charmhelpers/contrib/hardening/host/checks/limits.py deleted file mode 100644 index e94f5eb..0000000 --- a/hooks/charmhelpers/contrib/hardening/host/checks/limits.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2016 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from charmhelpers.contrib.hardening.audits.file import ( - DirectoryPermissionAudit, - TemplatedFile, -) -from charmhelpers.contrib.hardening.host import TEMPLATES_DIR -from charmhelpers.contrib.hardening import utils - - -def get_audits(): - """Get OS hardening security limits audits. - - :returns: dictionary of audits - """ - audits = [] - settings = utils.get_settings('os') - - # Ensure that the /etc/security/limits.d directory is only writable - # by the root user, but others can execute and read. - audits.append(DirectoryPermissionAudit('/etc/security/limits.d', - user='root', group='root', - mode=0o755)) - - # If core dumps are not enabled, then don't allow core dumps to be - # created as they may contain sensitive information. - if not settings['security']['kernel_enable_core_dump']: - audits.append(TemplatedFile('/etc/security/limits.d/10.hardcore.conf', - SecurityLimitsContext(), - template_dir=TEMPLATES_DIR, - user='root', group='root', mode=0o0440)) - return audits - - -class SecurityLimitsContext(object): - - def __call__(self): - settings = utils.get_settings('os') - ctxt = {'disable_core_dump': - not settings['security']['kernel_enable_core_dump']} - return ctxt diff --git a/hooks/charmhelpers/contrib/hardening/host/checks/login.py b/hooks/charmhelpers/contrib/hardening/host/checks/login.py deleted file mode 100644 index fe2bc6e..0000000 --- a/hooks/charmhelpers/contrib/hardening/host/checks/login.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2016 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from six import string_types - -from charmhelpers.contrib.hardening.audits.file import TemplatedFile -from charmhelpers.contrib.hardening.host import TEMPLATES_DIR -from charmhelpers.contrib.hardening import utils - - -def get_audits(): - """Get OS hardening login.defs audits. - - :returns: dictionary of audits - """ - audits = [TemplatedFile('/etc/login.defs', LoginContext(), - template_dir=TEMPLATES_DIR, - user='root', group='root', mode=0o0444)] - return audits - - -class LoginContext(object): - - def __call__(self): - settings = utils.get_settings('os') - - # Octal numbers in yaml end up being turned into decimal, - # so check if the umask is entered as a string (e.g. '027') - # or as an octal umask as we know it (e.g. 002). If its not - # a string assume it to be octal and turn it into an octal - # string. - umask = settings['environment']['umask'] - if not isinstance(umask, string_types): - umask = '%s' % oct(umask) - - ctxt = { - 'additional_user_paths': - settings['environment']['extra_user_paths'], - 'umask': umask, - 'pwd_max_age': settings['auth']['pw_max_age'], - 'pwd_min_age': settings['auth']['pw_min_age'], - 'uid_min': settings['auth']['uid_min'], - 'sys_uid_min': settings['auth']['sys_uid_min'], - 'sys_uid_max': settings['auth']['sys_uid_max'], - 'gid_min': settings['auth']['gid_min'], - 'sys_gid_min': settings['auth']['sys_gid_min'], - 'sys_gid_max': settings['auth']['sys_gid_max'], - 'login_retries': settings['auth']['retries'], - 'login_timeout': settings['auth']['timeout'], - 'chfn_restrict': settings['auth']['chfn_restrict'], - 'allow_login_without_home': settings['auth']['allow_homeless'] - } - - return ctxt diff --git a/hooks/charmhelpers/contrib/hardening/host/checks/minimize_access.py b/hooks/charmhelpers/contrib/hardening/host/checks/minimize_access.py deleted file mode 100644 index 6e64be0..0000000 --- a/hooks/charmhelpers/contrib/hardening/host/checks/minimize_access.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2016 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from charmhelpers.contrib.hardening.audits.file import ( - FilePermissionAudit, - ReadOnly, -) -from charmhelpers.contrib.hardening import utils - - -def get_audits(): - """Get OS hardening access audits. - - :returns: dictionary of audits - """ - audits = [] - settings = utils.get_settings('os') - - # Remove write permissions from $PATH folders for all regular users. - # This prevents changing system-wide commands from normal users. - path_folders = {'/usr/local/sbin', - '/usr/local/bin', - '/usr/sbin', - '/usr/bin', - '/bin'} - extra_user_paths = settings['environment']['extra_user_paths'] - path_folders.update(extra_user_paths) - audits.append(ReadOnly(path_folders)) - - # Only allow the root user to have access to the shadow file. - audits.append(FilePermissionAudit('/etc/shadow', 'root', 'root', 0o0600)) - - if 'change_user' not in settings['security']['users_allow']: - # su should only be accessible to user and group root, unless it is - # expressly defined to allow users to change to root via the - # security_users_allow config option. - audits.append(FilePermissionAudit('/bin/su', 'root', 'root', 0o750)) - - return audits diff --git a/hooks/charmhelpers/contrib/hardening/host/checks/pam.py b/hooks/charmhelpers/contrib/hardening/host/checks/pam.py deleted file mode 100644 index 9b38d5f..0000000 --- a/hooks/charmhelpers/contrib/hardening/host/checks/pam.py +++ /dev/null @@ -1,132 +0,0 @@ -# Copyright 2016 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from subprocess import ( - check_output, - CalledProcessError, -) - -from charmhelpers.core.hookenv import ( - log, - DEBUG, - ERROR, -) -from charmhelpers.fetch import ( - apt_install, - apt_purge, - apt_update, -) -from charmhelpers.contrib.hardening.audits.file import ( - TemplatedFile, - DeletedFile, -) -from charmhelpers.contrib.hardening import utils -from charmhelpers.contrib.hardening.host import TEMPLATES_DIR - - -def get_audits(): - """Get OS hardening PAM authentication audits. - - :returns: dictionary of audits - """ - audits = [] - - settings = utils.get_settings('os') - - if settings['auth']['pam_passwdqc_enable']: - audits.append(PasswdqcPAM('/etc/passwdqc.conf')) - - if settings['auth']['retries']: - audits.append(Tally2PAM('/usr/share/pam-configs/tally2')) - else: - audits.append(DeletedFile('/usr/share/pam-configs/tally2')) - - return audits - - -class PasswdqcPAMContext(object): - - def __call__(self): - ctxt = {} - settings = utils.get_settings('os') - - ctxt['auth_pam_passwdqc_options'] = \ - settings['auth']['pam_passwdqc_options'] - - return ctxt - - -class PasswdqcPAM(TemplatedFile): - """The PAM Audit verifies the linux PAM settings.""" - def __init__(self, path): - super(PasswdqcPAM, self).__init__(path=path, - template_dir=TEMPLATES_DIR, - context=PasswdqcPAMContext(), - user='root', - group='root', - mode=0o0640) - - def pre_write(self): - # Always remove? - for pkg in ['libpam-ccreds', 'libpam-cracklib']: - log("Purging package '%s'" % pkg, level=DEBUG), - apt_purge(pkg) - - apt_update(fatal=True) - for pkg in ['libpam-passwdqc']: - log("Installing package '%s'" % pkg, level=DEBUG), - apt_install(pkg) - - def post_write(self): - """Updates the PAM configuration after the file has been written""" - try: - check_output(['pam-auth-update', '--package']) - except CalledProcessError as e: - log('Error calling pam-auth-update: %s' % e, level=ERROR) - - -class Tally2PAMContext(object): - - def __call__(self): - ctxt = {} - settings = utils.get_settings('os') - - ctxt['auth_lockout_time'] = settings['auth']['lockout_time'] - ctxt['auth_retries'] = settings['auth']['retries'] - - return ctxt - - -class Tally2PAM(TemplatedFile): - """The PAM Audit verifies the linux PAM settings.""" - def __init__(self, path): - super(Tally2PAM, self).__init__(path=path, - template_dir=TEMPLATES_DIR, - context=Tally2PAMContext(), - user='root', - group='root', - mode=0o0640) - - def pre_write(self): - # Always remove? - apt_purge('libpam-ccreds') - apt_update(fatal=True) - apt_install('libpam-modules') - - def post_write(self): - """Updates the PAM configuration after the file has been written""" - try: - check_output(['pam-auth-update', '--package']) - except CalledProcessError as e: - log('Error calling pam-auth-update: %s' % e, level=ERROR) diff --git a/hooks/charmhelpers/contrib/hardening/host/checks/profile.py b/hooks/charmhelpers/contrib/hardening/host/checks/profile.py deleted file mode 100644 index 2727428..0000000 --- a/hooks/charmhelpers/contrib/hardening/host/checks/profile.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2016 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from charmhelpers.contrib.hardening.audits.file import TemplatedFile -from charmhelpers.contrib.hardening.host import TEMPLATES_DIR -from charmhelpers.contrib.hardening import utils - - -def get_audits(): - """Get OS hardening profile audits. - - :returns: dictionary of audits - """ - audits = [] - - settings = utils.get_settings('os') - # If core dumps are not enabled, then don't allow core dumps to be - # created as they may contain sensitive information. - if not settings['security']['kernel_enable_core_dump']: - audits.append(TemplatedFile('/etc/profile.d/pinerolo_profile.sh', - ProfileContext(), - template_dir=TEMPLATES_DIR, - mode=0o0755, user='root', group='root')) - if settings['security']['ssh_tmout']: - audits.append(TemplatedFile('/etc/profile.d/99-hardening.sh', - ProfileContext(), - template_dir=TEMPLATES_DIR, - mode=0o0644, user='root', group='root')) - return audits - - -class ProfileContext(object): - - def __call__(self): - settings = utils.get_settings('os') - ctxt = {'ssh_tmout': - settings['security']['ssh_tmout']} - return ctxt diff --git a/hooks/charmhelpers/contrib/hardening/host/checks/securetty.py b/hooks/charmhelpers/contrib/hardening/host/checks/securetty.py deleted file mode 100644 index 34cd021..0000000 --- a/hooks/charmhelpers/contrib/hardening/host/checks/securetty.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2016 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from charmhelpers.contrib.hardening.audits.file import TemplatedFile -from charmhelpers.contrib.hardening.host import TEMPLATES_DIR -from charmhelpers.contrib.hardening import utils - - -def get_audits(): - """Get OS hardening Secure TTY audits. - - :returns: dictionary of audits - """ - audits = [] - audits.append(TemplatedFile('/etc/securetty', SecureTTYContext(), - template_dir=TEMPLATES_DIR, - mode=0o0400, user='root', group='root')) - return audits - - -class SecureTTYContext(object): - - def __call__(self): - settings = utils.get_settings('os') - ctxt = {'ttys': settings['auth']['root_ttys']} - return ctxt diff --git a/hooks/charmhelpers/contrib/hardening/host/checks/suid_sgid.py b/hooks/charmhelpers/contrib/hardening/host/checks/suid_sgid.py deleted file mode 100644 index bcbe3fd..0000000 --- a/hooks/charmhelpers/contrib/hardening/host/checks/suid_sgid.py +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright 2016 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import subprocess - -from charmhelpers.core.hookenv import ( - log, - INFO, -) -from charmhelpers.contrib.hardening.audits.file import NoSUIDSGIDAudit -from charmhelpers.contrib.hardening import utils - - -BLACKLIST = ['/usr/bin/rcp', '/usr/bin/rlogin', '/usr/bin/rsh', - '/usr/libexec/openssh/ssh-keysign', - '/usr/lib/openssh/ssh-keysign', - '/sbin/netreport', - '/usr/sbin/usernetctl', - '/usr/sbin/userisdnctl', - '/usr/sbin/pppd', - '/usr/bin/lockfile', - '/usr/bin/mail-lock', - '/usr/bin/mail-unlock', - '/usr/bin/mail-touchlock', - '/usr/bin/dotlockfile', - '/usr/bin/arping', - '/usr/sbin/uuidd', - '/usr/bin/mtr', - '/usr/lib/evolution/camel-lock-helper-1.2', - '/usr/lib/pt_chown', - '/usr/lib/eject/dmcrypt-get-device', - '/usr/lib/mc/cons.saver'] - -WHITELIST = ['/bin/mount', '/bin/ping', '/bin/su', '/bin/umount', - '/sbin/pam_timestamp_check', '/sbin/unix_chkpwd', '/usr/bin/at', - '/usr/bin/gpasswd', '/usr/bin/locate', '/usr/bin/newgrp', - '/usr/bin/passwd', '/usr/bin/ssh-agent', - '/usr/libexec/utempter/utempter', '/usr/sbin/lockdev', - '/usr/sbin/sendmail.sendmail', '/usr/bin/expiry', - '/bin/ping6', '/usr/bin/traceroute6.iputils', - '/sbin/mount.nfs', '/sbin/umount.nfs', - '/sbin/mount.nfs4', '/sbin/umount.nfs4', - '/usr/bin/crontab', - '/usr/bin/wall', '/usr/bin/write', - '/usr/bin/screen', - '/usr/bin/mlocate', - '/usr/bin/chage', '/usr/bin/chfn', '/usr/bin/chsh', - '/bin/fusermount', - '/usr/bin/pkexec', - '/usr/bin/sudo', '/usr/bin/sudoedit', - '/usr/sbin/postdrop', '/usr/sbin/postqueue', - '/usr/sbin/suexec', - '/usr/lib/squid/ncsa_auth', '/usr/lib/squid/pam_auth', - '/usr/kerberos/bin/ksu', - '/usr/sbin/ccreds_validate', - '/usr/bin/Xorg', - '/usr/bin/X', - '/usr/lib/dbus-1.0/dbus-daemon-launch-helper', - '/usr/lib/vte/gnome-pty-helper', - '/usr/lib/libvte9/gnome-pty-helper', - '/usr/lib/libvte-2.90-9/gnome-pty-helper'] - - -def get_audits(): - """Get OS hardening suid/sgid audits. - - :returns: dictionary of audits - """ - checks = [] - settings = utils.get_settings('os') - if not settings['security']['suid_sgid_enforce']: - log("Skipping suid/sgid hardening", level=INFO) - return checks - - # Build the blacklist and whitelist of files for suid/sgid checks. - # There are a total of 4 lists: - # 1. the system blacklist - # 2. the system whitelist - # 3. the user blacklist - # 4. the user whitelist - # - # The blacklist is the set of paths which should NOT have the suid/sgid bit - # set and the whitelist is the set of paths which MAY have the suid/sgid - # bit setl. The user whitelist/blacklist effectively override the system - # whitelist/blacklist. - u_b = settings['security']['suid_sgid_blacklist'] - u_w = settings['security']['suid_sgid_whitelist'] - - blacklist = set(BLACKLIST) - set(u_w + u_b) - whitelist = set(WHITELIST) - set(u_b + u_w) - - checks.append(NoSUIDSGIDAudit(blacklist)) - - dry_run = settings['security']['suid_sgid_dry_run_on_unknown'] - - if settings['security']['suid_sgid_remove_from_unknown'] or dry_run: - # If the policy is a dry_run (e.g. complain only) or remove unknown - # suid/sgid bits then find all of the paths which have the suid/sgid - # bit set and then remove the whitelisted paths. - root_path = settings['environment']['root_path'] - unknown_paths = find_paths_with_suid_sgid(root_path) - set(whitelist) - checks.append(NoSUIDSGIDAudit(unknown_paths, unless=dry_run)) - - return checks - - -def find_paths_with_suid_sgid(root_path): - """Finds all paths/files which have an suid/sgid bit enabled. - - Starting with the root_path, this will recursively find all paths which - have an suid or sgid bit set. - """ - cmd = ['find', root_path, '-perm', '-4000', '-o', '-perm', '-2000', - '-type', 'f', '!', '-path', '/proc/*', '-print'] - - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, _ = p.communicate() - return set(out.split('\n')) diff --git a/hooks/charmhelpers/contrib/hardening/host/checks/sysctl.py b/hooks/charmhelpers/contrib/hardening/host/checks/sysctl.py deleted file mode 100644 index f1ea581..0000000 --- a/hooks/charmhelpers/contrib/hardening/host/checks/sysctl.py +++ /dev/null @@ -1,209 +0,0 @@ -# Copyright 2016 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import platform -import re -import six -import subprocess - -from charmhelpers.core.hookenv import ( - log, - INFO, - WARNING, -) -from charmhelpers.contrib.hardening import utils -from charmhelpers.contrib.hardening.audits.file import ( - FilePermissionAudit, - TemplatedFile, -) -from charmhelpers.contrib.hardening.host import TEMPLATES_DIR - - -SYSCTL_DEFAULTS = """net.ipv4.ip_forward=%(net_ipv4_ip_forward)s -net.ipv6.conf.all.forwarding=%(net_ipv6_conf_all_forwarding)s -net.ipv4.conf.all.rp_filter=1 -net.ipv4.conf.default.rp_filter=1 -net.ipv4.icmp_echo_ignore_broadcasts=1 -net.ipv4.icmp_ignore_bogus_error_responses=1 -net.ipv4.icmp_ratelimit=100 -net.ipv4.icmp_ratemask=88089 -net.ipv6.conf.all.disable_ipv6=%(net_ipv6_conf_all_disable_ipv6)s -net.ipv4.tcp_timestamps=%(net_ipv4_tcp_timestamps)s -net.ipv4.conf.all.arp_ignore=%(net_ipv4_conf_all_arp_ignore)s -net.ipv4.conf.all.arp_announce=%(net_ipv4_conf_all_arp_announce)s -net.ipv4.tcp_rfc1337=1 -net.ipv4.tcp_syncookies=1 -net.ipv4.conf.all.shared_media=1 -net.ipv4.conf.default.shared_media=1 -net.ipv4.conf.all.accept_source_route=0 -net.ipv4.conf.default.accept_source_route=0 -net.ipv4.conf.all.accept_redirects=0 -net.ipv4.conf.default.accept_redirects=0 -net.ipv6.conf.all.accept_redirects=0 -net.ipv6.conf.default.accept_redirects=0 -net.ipv4.conf.all.secure_redirects=0 -net.ipv4.conf.default.secure_redirects=0 -net.ipv4.conf.all.send_redirects=0 -net.ipv4.conf.default.send_redirects=0 -net.ipv4.conf.all.log_martians=0 -net.ipv6.conf.default.router_solicitations=0 -net.ipv6.conf.default.accept_ra_rtr_pref=0 -net.ipv6.conf.default.accept_ra_pinfo=0 -net.ipv6.conf.default.accept_ra_defrtr=0 -net.ipv6.conf.default.autoconf=0 -net.ipv6.conf.default.dad_transmits=0 -net.ipv6.conf.default.max_addresses=1 -net.ipv6.conf.all.accept_ra=0 -net.ipv6.conf.default.accept_ra=0 -kernel.modules_disabled=%(kernel_modules_disabled)s -kernel.sysrq=%(kernel_sysrq)s -fs.suid_dumpable=%(fs_suid_dumpable)s -kernel.randomize_va_space=2 -""" - - -def get_audits(): - """Get OS hardening sysctl audits. - - :returns: dictionary of audits - """ - audits = [] - settings = utils.get_settings('os') - - # Apply the sysctl settings which are configured to be applied. - audits.append(SysctlConf()) - # Make sure that only root has access to the sysctl.conf file, and - # that it is read-only. - audits.append(FilePermissionAudit('/etc/sysctl.conf', - user='root', - group='root', mode=0o0440)) - # If module loading is not enabled, then ensure that the modules - # file has the appropriate permissions and rebuild the initramfs - if not settings['security']['kernel_enable_module_loading']: - audits.append(ModulesTemplate()) - - return audits - - -class ModulesContext(object): - - def __call__(self): - settings = utils.get_settings('os') - with open('/proc/cpuinfo', 'r') as fd: - cpuinfo = fd.readlines() - - for line in cpuinfo: - match = re.search(r"^vendor_id\s+:\s+(.+)", line) - if match: - vendor = match.group(1) - - if vendor == "GenuineIntel": - vendor = "intel" - elif vendor == "AuthenticAMD": - vendor = "amd" - - ctxt = {'arch': platform.processor(), - 'cpuVendor': vendor, - 'desktop_enable': settings['general']['desktop_enable']} - - return ctxt - - -class ModulesTemplate(object): - - def __init__(self): - super(ModulesTemplate, self).__init__('/etc/initramfs-tools/modules', - ModulesContext(), - templates_dir=TEMPLATES_DIR, - user='root', group='root', - mode=0o0440) - - def post_write(self): - subprocess.check_call(['update-initramfs', '-u']) - - -class SysCtlHardeningContext(object): - def __call__(self): - settings = utils.get_settings('os') - ctxt = {'sysctl': {}} - - log("Applying sysctl settings", level=INFO) - extras = {'net_ipv4_ip_forward': 0, - 'net_ipv6_conf_all_forwarding': 0, - 'net_ipv6_conf_all_disable_ipv6': 1, - 'net_ipv4_tcp_timestamps': 0, - 'net_ipv4_conf_all_arp_ignore': 0, - 'net_ipv4_conf_all_arp_announce': 0, - 'kernel_sysrq': 0, - 'fs_suid_dumpable': 0, - 'kernel_modules_disabled': 1} - - if settings['sysctl']['ipv6_enable']: - extras['net_ipv6_conf_all_disable_ipv6'] = 0 - - if settings['sysctl']['forwarding']: - extras['net_ipv4_ip_forward'] = 1 - extras['net_ipv6_conf_all_forwarding'] = 1 - - if settings['sysctl']['arp_restricted']: - extras['net_ipv4_conf_all_arp_ignore'] = 1 - extras['net_ipv4_conf_all_arp_announce'] = 2 - - if settings['security']['kernel_enable_module_loading']: - extras['kernel_modules_disabled'] = 0 - - if settings['sysctl']['kernel_enable_sysrq']: - sysrq_val = settings['sysctl']['kernel_secure_sysrq'] - extras['kernel_sysrq'] = sysrq_val - - if settings['security']['kernel_enable_core_dump']: - extras['fs_suid_dumpable'] = 1 - - settings.update(extras) - for d in (SYSCTL_DEFAULTS % settings).split(): - d = d.strip().partition('=') - key = d[0].strip() - path = os.path.join('/proc/sys', key.replace('.', '/')) - if not os.path.exists(path): - log("Skipping '%s' since '%s' does not exist" % (key, path), - level=WARNING) - continue - - ctxt['sysctl'][key] = d[2] or None - - # Translate for python3 - return {'sysctl_settings': - [(k, v) for k, v in six.iteritems(ctxt['sysctl'])]} - - -class SysctlConf(TemplatedFile): - """An audit check for sysctl settings.""" - def __init__(self): - self.conffile = '/etc/sysctl.d/99-juju-hardening.conf' - super(SysctlConf, self).__init__(self.conffile, - SysCtlHardeningContext(), - template_dir=TEMPLATES_DIR, - user='root', group='root', - mode=0o0440) - - def post_write(self): - try: - subprocess.check_call(['sysctl', '-p', self.conffile]) - except subprocess.CalledProcessError as e: - # NOTE: on some systems if sysctl cannot apply all settings it - # will return non-zero as well. - log("sysctl command returned an error (maybe some " - "keys could not be set) - %s" % (e), - level=WARNING) diff --git a/hooks/charmhelpers/contrib/hardening/host/templates/10.hardcore.conf b/hooks/charmhelpers/contrib/hardening/host/templates/10.hardcore.conf deleted file mode 100644 index 0014191..0000000 --- a/hooks/charmhelpers/contrib/hardening/host/templates/10.hardcore.conf +++ /dev/null @@ -1,8 +0,0 @@ -############################################################################### -# WARNING: This configuration file is maintained by Juju. Local changes may -# be overwritten. -############################################################################### -{% if disable_core_dump -%} -# Prevent core dumps for all users. These are usually only needed by developers and may contain sensitive information. -* hard core 0 -{% endif %} \ No newline at end of file diff --git a/hooks/charmhelpers/contrib/hardening/host/templates/99-hardening.sh b/hooks/charmhelpers/contrib/hardening/host/templates/99-hardening.sh deleted file mode 100644 index 616cef4..0000000 --- a/hooks/charmhelpers/contrib/hardening/host/templates/99-hardening.sh +++ /dev/null @@ -1,5 +0,0 @@ -TMOUT={{ tmout }} -readonly TMOUT -export TMOUT - -readonly HISTFILE diff --git a/hooks/charmhelpers/contrib/hardening/host/templates/99-juju-hardening.conf b/hooks/charmhelpers/contrib/hardening/host/templates/99-juju-hardening.conf deleted file mode 100644 index 101f1e1..0000000 --- a/hooks/charmhelpers/contrib/hardening/host/templates/99-juju-hardening.conf +++ /dev/null @@ -1,7 +0,0 @@ -############################################################################### -# WARNING: This configuration file is maintained by Juju. Local changes may -# be overwritten. -############################################################################### -{% for key, value in sysctl_settings -%} -{{ key }}={{ value }} -{% endfor -%} diff --git a/hooks/charmhelpers/contrib/hardening/host/templates/__init__.py b/hooks/charmhelpers/contrib/hardening/host/templates/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/hooks/charmhelpers/contrib/hardening/host/templates/login.defs b/hooks/charmhelpers/contrib/hardening/host/templates/login.defs deleted file mode 100644 index db137d6..0000000 --- a/hooks/charmhelpers/contrib/hardening/host/templates/login.defs +++ /dev/null @@ -1,349 +0,0 @@ -############################################################################### -# WARNING: This configuration file is maintained by Juju. Local changes may -# be overwritten. -############################################################################### -# -# /etc/login.defs - Configuration control definitions for the login package. -# -# Three items must be defined: MAIL_DIR, ENV_SUPATH, and ENV_PATH. -# If unspecified, some arbitrary (and possibly incorrect) value will -# be assumed. All other items are optional - if not specified then -# the described action or option will be inhibited. -# -# Comment lines (lines beginning with "#") and blank lines are ignored. -# -# Modified for Linux. --marekm - -# REQUIRED for useradd/userdel/usermod -# Directory where mailboxes reside, _or_ name of file, relative to the -# home directory. If you _do_ define MAIL_DIR and MAIL_FILE, -# MAIL_DIR takes precedence. -# -# Essentially: -# - MAIL_DIR defines the location of users mail spool files -# (for mbox use) by appending the username to MAIL_DIR as defined -# below. -# - MAIL_FILE defines the location of the users mail spool files as the -# fully-qualified filename obtained by prepending the user home -# directory before $MAIL_FILE -# -# NOTE: This is no more used for setting up users MAIL environment variable -# which is, starting from shadow 4.0.12-1 in Debian, entirely the -# job of the pam_mail PAM modules -# See default PAM configuration files provided for -# login, su, etc. -# -# This is a temporary situation: setting these variables will soon -# move to /etc/default/useradd and the variables will then be -# no more supported -MAIL_DIR /var/mail -#MAIL_FILE .mail - -# -# Enable logging and display of /var/log/faillog login failure info. -# This option conflicts with the pam_tally PAM module. -# -FAILLOG_ENAB yes - -# -# Enable display of unknown usernames when login failures are recorded. -# -# WARNING: Unknown usernames may become world readable. -# See #290803 and #298773 for details about how this could become a security -# concern -LOG_UNKFAIL_ENAB no - -# -# Enable logging of successful logins -# -LOG_OK_LOGINS yes - -# -# Enable "syslog" logging of su activity - in addition to sulog file logging. -# SYSLOG_SG_ENAB does the same for newgrp and sg. -# -SYSLOG_SU_ENAB yes -SYSLOG_SG_ENAB yes - -# -# If defined, all su activity is logged to this file. -# -#SULOG_FILE /var/log/sulog - -# -# If defined, file which maps tty line to TERM environment parameter. -# Each line of the file is in a format something like "vt100 tty01". -# -#TTYTYPE_FILE /etc/ttytype - -# -# If defined, login failures will be logged here in a utmp format -# last, when invoked as lastb, will read /var/log/btmp, so... -# -FTMP_FILE /var/log/btmp - -# -# If defined, the command name to display when running "su -". For -# example, if this is defined as "su" then a "ps" will display the -# command is "-su". If not defined, then "ps" would display the -# name of the shell actually being run, e.g. something like "-sh". -# -SU_NAME su - -# -# If defined, file which inhibits all the usual chatter during the login -# sequence. If a full pathname, then hushed mode will be enabled if the -# user's name or shell are found in the file. If not a full pathname, then -# hushed mode will be enabled if the file exists in the user's home directory. -# -HUSHLOGIN_FILE .hushlogin -#HUSHLOGIN_FILE /etc/hushlogins - -# -# *REQUIRED* The default PATH settings, for superuser and normal users. -# -# (they are minimal, add the rest in the shell startup files) -ENV_SUPATH PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin -ENV_PATH PATH=/usr/local/bin:/usr/bin:/bin{% if additional_user_paths %}{{ additional_user_paths }}{% endif %} - -# -# Terminal permissions -# -# TTYGROUP Login tty will be assigned this group ownership. -# TTYPERM Login tty will be set to this permission. -# -# If you have a "write" program which is "setgid" to a special group -# which owns the terminals, define TTYGROUP to the group number and -# TTYPERM to 0620. Otherwise leave TTYGROUP commented out and assign -# TTYPERM to either 622 or 600. -# -# In Debian /usr/bin/bsd-write or similar programs are setgid tty -# However, the default and recommended value for TTYPERM is still 0600 -# to not allow anyone to write to anyone else console or terminal - -# Users can still allow other people to write them by issuing -# the "mesg y" command. - -TTYGROUP tty -TTYPERM 0600 - -# -# Login configuration initializations: -# -# ERASECHAR Terminal ERASE character ('\010' = backspace). -# KILLCHAR Terminal KILL character ('\025' = CTRL/U). -# UMASK Default "umask" value. -# -# The ERASECHAR and KILLCHAR are used only on System V machines. -# -# UMASK is the default umask value for pam_umask and is used by -# useradd and newusers to set the mode of the new home directories. -# 022 is the "historical" value in Debian for UMASK -# 027, or even 077, could be considered better for privacy -# There is no One True Answer here : each sysadmin must make up his/her -# mind. -# -# If USERGROUPS_ENAB is set to "yes", that will modify this UMASK default value -# for private user groups, i. e. the uid is the same as gid, and username is -# the same as the primary group name: for these, the user permissions will be -# used as group permissions, e. g. 022 will become 002. -# -# Prefix these values with "0" to get octal, "0x" to get hexadecimal. -# -ERASECHAR 0177 -KILLCHAR 025 -UMASK {{ umask }} - -# Enable setting of the umask group bits to be the same as owner bits (examples: `022` -> `002`, `077` -> `007`) for non-root users, if the uid is the same as gid, and username is the same as the primary group name. -# If set to yes, userdel will remove the user´s group if it contains no more members, and useradd will create by default a group with the name of the user. -USERGROUPS_ENAB yes - -# -# Password aging controls: -# -# PASS_MAX_DAYS Maximum number of days a password may be used. -# PASS_MIN_DAYS Minimum number of days allowed between password changes. -# PASS_WARN_AGE Number of days warning given before a password expires. -# -PASS_MAX_DAYS {{ pwd_max_age }} -PASS_MIN_DAYS {{ pwd_min_age }} -PASS_WARN_AGE 7 - -# -# Min/max values for automatic uid selection in useradd -# -UID_MIN {{ uid_min }} -UID_MAX 60000 -# System accounts -SYS_UID_MIN {{ sys_uid_min }} -SYS_UID_MAX {{ sys_uid_max }} - -# Min/max values for automatic gid selection in groupadd -GID_MIN {{ gid_min }} -GID_MAX 60000 -# System accounts -SYS_GID_MIN {{ sys_gid_min }} -SYS_GID_MAX {{ sys_gid_max }} - -# -# Max number of login retries if password is bad. This will most likely be -# overriden by PAM, since the default pam_unix module has it's own built -# in of 3 retries. However, this is a safe fallback in case you are using -# an authentication module that does not enforce PAM_MAXTRIES. -# -LOGIN_RETRIES {{ login_retries }} - -# -# Max time in seconds for login -# -LOGIN_TIMEOUT {{ login_timeout }} - -# -# Which fields may be changed by regular users using chfn - use -# any combination of letters "frwh" (full name, room number, work -# phone, home phone). If not defined, no changes are allowed. -# For backward compatibility, "yes" = "rwh" and "no" = "frwh". -# -{% if chfn_restrict %} -CHFN_RESTRICT {{ chfn_restrict }} -{% endif %} - -# -# Should login be allowed if we can't cd to the home directory? -# Default in no. -# -DEFAULT_HOME {% if allow_login_without_home %} yes {% else %} no {% endif %} - -# -# If defined, this command is run when removing a user. -# It should remove any at/cron/print jobs etc. owned by -# the user to be removed (passed as the first argument). -# -#USERDEL_CMD /usr/sbin/userdel_local - -# -# Enable setting of the umask group bits to be the same as owner bits -# (examples: 022 -> 002, 077 -> 007) for non-root users, if the uid is -# the same as gid, and username is the same as the primary group name. -# -# If set to yes, userdel will remove the user´s group if it contains no -# more members, and useradd will create by default a group with the name -# of the user. -# -USERGROUPS_ENAB yes - -# -# Instead of the real user shell, the program specified by this parameter -# will be launched, although its visible name (argv[0]) will be the shell's. -# The program may do whatever it wants (logging, additional authentification, -# banner, ...) before running the actual shell. -# -# FAKE_SHELL /bin/fakeshell - -# -# If defined, either full pathname of a file containing device names or -# a ":" delimited list of device names. Root logins will be allowed only -# upon these devices. -# -# This variable is used by login and su. -# -#CONSOLE /etc/consoles -#CONSOLE console:tty01:tty02:tty03:tty04 - -# -# List of groups to add to the user's supplementary group set -# when logging in on the console (as determined by the CONSOLE -# setting). Default is none. -# -# Use with caution - it is possible for users to gain permanent -# access to these groups, even when not logged in on the console. -# How to do it is left as an exercise for the reader... -# -# This variable is used by login and su. -# -#CONSOLE_GROUPS floppy:audio:cdrom - -# -# If set to "yes", new passwords will be encrypted using the MD5-based -# algorithm compatible with the one used by recent releases of FreeBSD. -# It supports passwords of unlimited length and longer salt strings. -# Set to "no" if you need to copy encrypted passwords to other systems -# which don't understand the new algorithm. Default is "no". -# -# This variable is deprecated. You should use ENCRYPT_METHOD. -# -MD5_CRYPT_ENAB no - -# -# If set to MD5 , MD5-based algorithm will be used for encrypting password -# If set to SHA256, SHA256-based algorithm will be used for encrypting password -# If set to SHA512, SHA512-based algorithm will be used for encrypting password -# If set to DES, DES-based algorithm will be used for encrypting password (default) -# Overrides the MD5_CRYPT_ENAB option -# -# Note: It is recommended to use a value consistent with -# the PAM modules configuration. -# -ENCRYPT_METHOD SHA512 - -# -# Only used if ENCRYPT_METHOD is set to SHA256 or SHA512. -# -# Define the number of SHA rounds. -# With a lot of rounds, it is more difficult to brute forcing the password. -# But note also that it more CPU resources will be needed to authenticate -# users. -# -# If not specified, the libc will choose the default number of rounds (5000). -# The values must be inside the 1000-999999999 range. -# If only one of the MIN or MAX values is set, then this value will be used. -# If MIN > MAX, the highest value will be used. -# -# SHA_CRYPT_MIN_ROUNDS 5000 -# SHA_CRYPT_MAX_ROUNDS 5000 - -################# OBSOLETED BY PAM ############## -# # -# These options are now handled by PAM. Please # -# edit the appropriate file in /etc/pam.d/ to # -# enable the equivelants of them. -# -############### - -#MOTD_FILE -#DIALUPS_CHECK_ENAB -#LASTLOG_ENAB -#MAIL_CHECK_ENAB -#OBSCURE_CHECKS_ENAB -#PORTTIME_CHECKS_ENAB -#SU_WHEEL_ONLY -#CRACKLIB_DICTPATH -#PASS_CHANGE_TRIES -#PASS_ALWAYS_WARN -#ENVIRON_FILE -#NOLOGINS_FILE -#ISSUE_FILE -#PASS_MIN_LEN -#PASS_MAX_LEN -#ULIMIT -#ENV_HZ -#CHFN_AUTH -#CHSH_AUTH -#FAIL_DELAY - -################# OBSOLETED ####################### -# # -# These options are no more handled by shadow. # -# # -# Shadow utilities will display a warning if they # -# still appear. # -# # -################################################### - -# CLOSE_SESSIONS -# LOGIN_STRING -# NO_PASSWORD_CONSOLE -# QMAIL_DIR - - - diff --git a/hooks/charmhelpers/contrib/hardening/host/templates/modules b/hooks/charmhelpers/contrib/hardening/host/templates/modules deleted file mode 100644 index ef0354e..0000000 --- a/hooks/charmhelpers/contrib/hardening/host/templates/modules +++ /dev/null @@ -1,117 +0,0 @@ -############################################################################### -# WARNING: This configuration file is maintained by Juju. Local changes may -# be overwritten. -############################################################################### -# /etc/modules: kernel modules to load at boot time. -# -# This file contains the names of kernel modules that should be loaded -# at boot time, one per line. Lines beginning with "#" are ignored. -# Parameters can be specified after the module name. - -# Arch -# ---- -# -# Modules for certains builds, contains support modules and some CPU-specific optimizations. - -{% if arch == "x86_64" -%} -# Optimize for x86_64 cryptographic features -twofish-x86_64-3way -twofish-x86_64 -aes-x86_64 -salsa20-x86_64 -blowfish-x86_64 -{% endif -%} - -{% if cpuVendor == "intel" -%} -# Intel-specific optimizations -ghash-clmulni-intel -aesni-intel -kvm-intel -{% endif -%} - -{% if cpuVendor == "amd" -%} -# AMD-specific optimizations -kvm-amd -{% endif -%} - -kvm - - -# Crypto -# ------ - -# Some core modules which comprise strong cryptography. -blowfish_common -blowfish_generic -ctr -cts -lrw -lzo -rmd160 -rmd256 -rmd320 -serpent -sha512_generic -twofish_common -twofish_generic -xts -zlib - - -# Drivers -# ------- - -# Basics -lp -rtc -loop - -# Filesystems -ext2 -btrfs - -{% if desktop_enable -%} -# Desktop -psmouse -snd -snd_ac97_codec -snd_intel8x0 -snd_page_alloc -snd_pcm -snd_timer -soundcore -usbhid -{% endif -%} - -# Lib -# --- -xz - - -# Net -# --- - -# All packets needed for netfilter rules (ie iptables, ebtables). -ip_tables -x_tables -iptable_filter -iptable_nat - -# Targets -ipt_LOG -ipt_REJECT - -# Modules -xt_connlimit -xt_tcpudp -xt_recent -xt_limit -xt_conntrack -nf_conntrack -nf_conntrack_ipv4 -nf_defrag_ipv4 -xt_state -nf_nat - -# Addons -xt_pknock \ No newline at end of file diff --git a/hooks/charmhelpers/contrib/hardening/host/templates/passwdqc.conf b/hooks/charmhelpers/contrib/hardening/host/templates/passwdqc.conf deleted file mode 100644 index f98d14e..0000000 --- a/hooks/charmhelpers/contrib/hardening/host/templates/passwdqc.conf +++ /dev/null @@ -1,11 +0,0 @@ -############################################################################### -# WARNING: This configuration file is maintained by Juju. Local changes may -# be overwritten. -############################################################################### -Name: passwdqc password strength enforcement -Default: yes -Priority: 1024 -Conflicts: cracklib -Password-Type: Primary -Password: - requisite pam_passwdqc.so {{ auth_pam_passwdqc_options }} diff --git a/hooks/charmhelpers/contrib/hardening/host/templates/pinerolo_profile.sh b/hooks/charmhelpers/contrib/hardening/host/templates/pinerolo_profile.sh deleted file mode 100644 index fd2de79..0000000 --- a/hooks/charmhelpers/contrib/hardening/host/templates/pinerolo_profile.sh +++ /dev/null @@ -1,8 +0,0 @@ -############################################################################### -# WARNING: This configuration file is maintained by Juju. Local changes may -# be overwritten. -############################################################################### -# Disable core dumps via soft limits for all users. Compliance to this setting -# is voluntary and can be modified by users up to a hard limit. This setting is -# a sane default. -ulimit -S -c 0 > /dev/null 2>&1 diff --git a/hooks/charmhelpers/contrib/hardening/host/templates/securetty b/hooks/charmhelpers/contrib/hardening/host/templates/securetty deleted file mode 100644 index 15b18d4..0000000 --- a/hooks/charmhelpers/contrib/hardening/host/templates/securetty +++ /dev/null @@ -1,11 +0,0 @@ -############################################################################### -# WARNING: This configuration file is maintained by Juju. Local changes may -# be overwritten. -############################################################################### -# A list of TTYs, from which root can log in -# see `man securetty` for reference -{% if ttys -%} -{% for tty in ttys -%} -{{ tty }} -{% endfor -%} -{% endif -%} diff --git a/hooks/charmhelpers/contrib/hardening/host/templates/tally2 b/hooks/charmhelpers/contrib/hardening/host/templates/tally2 deleted file mode 100644 index d962029..0000000 --- a/hooks/charmhelpers/contrib/hardening/host/templates/tally2 +++ /dev/null @@ -1,14 +0,0 @@ -############################################################################### -# WARNING: This configuration file is maintained by Juju. Local changes may -# be overwritten. -############################################################################### -Name: tally2 lockout after failed attempts enforcement -Default: yes -Priority: 1024 -Conflicts: cracklib -Auth-Type: Primary -Auth-Initial: - required pam_tally2.so deny={{ auth_retries }} onerr=fail unlock_time={{ auth_lockout_time }} -Account-Type: Primary -Account-Initial: - required pam_tally2.so diff --git a/hooks/charmhelpers/contrib/hardening/mysql/__init__.py b/hooks/charmhelpers/contrib/hardening/mysql/__init__.py deleted file mode 100644 index 58bebd8..0000000 --- a/hooks/charmhelpers/contrib/hardening/mysql/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2016 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from os import path - -TEMPLATES_DIR = path.join(path.dirname(__file__), 'templates') diff --git a/hooks/charmhelpers/contrib/hardening/mysql/checks/__init__.py b/hooks/charmhelpers/contrib/hardening/mysql/checks/__init__.py deleted file mode 100644 index 1990d85..0000000 --- a/hooks/charmhelpers/contrib/hardening/mysql/checks/__init__.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2016 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from charmhelpers.core.hookenv import ( - log, - DEBUG, -) -from charmhelpers.contrib.hardening.mysql.checks import config - - -def run_mysql_checks(): - log("Starting MySQL hardening checks.", level=DEBUG) - checks = config.get_audits() - for check in checks: - log("Running '%s' check" % (check.__class__.__name__), level=DEBUG) - check.ensure_compliance() - - log("MySQL hardening checks complete.", level=DEBUG) diff --git a/hooks/charmhelpers/contrib/hardening/mysql/checks/config.py b/hooks/charmhelpers/contrib/hardening/mysql/checks/config.py deleted file mode 100644 index a79f33b..0000000 --- a/hooks/charmhelpers/contrib/hardening/mysql/checks/config.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright 2016 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import six -import subprocess - -from charmhelpers.core.hookenv import ( - log, - WARNING, -) -from charmhelpers.contrib.hardening.audits.file import ( - FilePermissionAudit, - DirectoryPermissionAudit, - TemplatedFile, -) -from charmhelpers.contrib.hardening.mysql import TEMPLATES_DIR -from charmhelpers.contrib.hardening import utils - - -def get_audits(): - """Get MySQL hardening config audits. - - :returns: dictionary of audits - """ - if subprocess.call(['which', 'mysql'], stdout=subprocess.PIPE) != 0: - log("MySQL does not appear to be installed on this node - " - "skipping mysql hardening", level=WARNING) - return [] - - settings = utils.get_settings('mysql') - hardening_settings = settings['hardening'] - my_cnf = hardening_settings['mysql-conf'] - - audits = [ - FilePermissionAudit(paths=[my_cnf], user='root', - group='root', mode=0o0600), - - TemplatedFile(hardening_settings['hardening-conf'], - MySQLConfContext(), - TEMPLATES_DIR, - mode=0o0750, - user='mysql', - group='root', - service_actions=[{'service': 'mysql', - 'actions': ['restart']}]), - - # MySQL and Percona charms do not allow configuration of the - # data directory, so use the default. - DirectoryPermissionAudit('/var/lib/mysql', - user='mysql', - group='mysql', - recursive=False, - mode=0o755), - - DirectoryPermissionAudit('/etc/mysql', - user='root', - group='root', - recursive=False, - mode=0o700), - ] - - return audits - - -class MySQLConfContext(object): - """Defines the set of key/value pairs to set in a mysql config file. - - This context, when called, will return a dictionary containing the - key/value pairs of setting to specify in the - /etc/mysql/conf.d/hardening.cnf file. - """ - def __call__(self): - settings = utils.get_settings('mysql') - # Translate for python3 - return {'mysql_settings': - [(k, v) for k, v in six.iteritems(settings['security'])]} diff --git a/hooks/charmhelpers/contrib/hardening/mysql/templates/__init__.py b/hooks/charmhelpers/contrib/hardening/mysql/templates/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/hooks/charmhelpers/contrib/hardening/mysql/templates/hardening.cnf b/hooks/charmhelpers/contrib/hardening/mysql/templates/hardening.cnf deleted file mode 100644 index 8242586..0000000 --- a/hooks/charmhelpers/contrib/hardening/mysql/templates/hardening.cnf +++ /dev/null @@ -1,12 +0,0 @@ -############################################################################### -# WARNING: This configuration file is maintained by Juju. Local changes may -# be overwritten. -############################################################################### -[mysqld] -{% for setting, value in mysql_settings -%} -{% if value == 'True' -%} -{{ setting }} -{% elif value != 'None' and value != None -%} -{{ setting }} = {{ value }} -{% endif -%} -{% endfor -%} diff --git a/hooks/charmhelpers/contrib/hardening/ssh/__init__.py b/hooks/charmhelpers/contrib/hardening/ssh/__init__.py deleted file mode 100644 index 58bebd8..0000000 --- a/hooks/charmhelpers/contrib/hardening/ssh/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2016 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from os import path - -TEMPLATES_DIR = path.join(path.dirname(__file__), 'templates') diff --git a/hooks/charmhelpers/contrib/hardening/ssh/checks/__init__.py b/hooks/charmhelpers/contrib/hardening/ssh/checks/__init__.py deleted file mode 100644 index edaf484..0000000 --- a/hooks/charmhelpers/contrib/hardening/ssh/checks/__init__.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2016 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from charmhelpers.core.hookenv import ( - log, - DEBUG, -) -from charmhelpers.contrib.hardening.ssh.checks import config - - -def run_ssh_checks(): - log("Starting SSH hardening checks.", level=DEBUG) - checks = config.get_audits() - for check in checks: - log("Running '%s' check" % (check.__class__.__name__), level=DEBUG) - check.ensure_compliance() - - log("SSH hardening checks complete.", level=DEBUG) diff --git a/hooks/charmhelpers/contrib/hardening/ssh/checks/config.py b/hooks/charmhelpers/contrib/hardening/ssh/checks/config.py deleted file mode 100644 index 41bed2d..0000000 --- a/hooks/charmhelpers/contrib/hardening/ssh/checks/config.py +++ /dev/null @@ -1,435 +0,0 @@ -# Copyright 2016 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -from charmhelpers.contrib.network.ip import ( - get_address_in_network, - get_iface_addr, - is_ip, -) -from charmhelpers.core.hookenv import ( - log, - DEBUG, -) -from charmhelpers.fetch import ( - apt_install, - apt_update, -) -from charmhelpers.core.host import ( - lsb_release, - CompareHostReleases, -) -from charmhelpers.contrib.hardening.audits.file import ( - TemplatedFile, - FileContentAudit, -) -from charmhelpers.contrib.hardening.ssh import TEMPLATES_DIR -from charmhelpers.contrib.hardening import utils - - -def get_audits(): - """Get SSH hardening config audits. - - :returns: dictionary of audits - """ - audits = [SSHConfig(), SSHDConfig(), SSHConfigFileContentAudit(), - SSHDConfigFileContentAudit()] - return audits - - -class SSHConfigContext(object): - - type = 'client' - - def get_macs(self, allow_weak_mac): - if allow_weak_mac: - weak_macs = 'weak' - else: - weak_macs = 'default' - - default = 'hmac-sha2-512,hmac-sha2-256,hmac-ripemd160' - macs = {'default': default, - 'weak': default + ',hmac-sha1'} - - default = ('hmac-sha2-512-etm@openssh.com,' - 'hmac-sha2-256-etm@openssh.com,' - 'hmac-ripemd160-etm@openssh.com,umac-128-etm@openssh.com,' - 'hmac-sha2-512,hmac-sha2-256,hmac-ripemd160') - macs_66 = {'default': default, - 'weak': default + ',hmac-sha1'} - - # Use newer ciphers on Ubuntu Trusty and above - _release = lsb_release()['DISTRIB_CODENAME'].lower() - if CompareHostReleases(_release) >= 'trusty': - log("Detected Ubuntu 14.04 or newer, using new macs", level=DEBUG) - macs = macs_66 - - return macs[weak_macs] - - def get_kexs(self, allow_weak_kex): - if allow_weak_kex: - weak_kex = 'weak' - else: - weak_kex = 'default' - - default = 'diffie-hellman-group-exchange-sha256' - weak = (default + ',diffie-hellman-group14-sha1,' - 'diffie-hellman-group-exchange-sha1,' - 'diffie-hellman-group1-sha1') - kex = {'default': default, - 'weak': weak} - - default = ('curve25519-sha256@libssh.org,' - 'diffie-hellman-group-exchange-sha256') - weak = (default + ',diffie-hellman-group14-sha1,' - 'diffie-hellman-group-exchange-sha1,' - 'diffie-hellman-group1-sha1') - kex_66 = {'default': default, - 'weak': weak} - - # Use newer kex on Ubuntu Trusty and above - _release = lsb_release()['DISTRIB_CODENAME'].lower() - if CompareHostReleases(_release) >= 'trusty': - log('Detected Ubuntu 14.04 or newer, using new key exchange ' - 'algorithms', level=DEBUG) - kex = kex_66 - - return kex[weak_kex] - - def get_ciphers(self, cbc_required): - if cbc_required: - weak_ciphers = 'weak' - else: - weak_ciphers = 'default' - - default = 'aes256-ctr,aes192-ctr,aes128-ctr' - cipher = {'default': default, - 'weak': default + 'aes256-cbc,aes192-cbc,aes128-cbc'} - - default = ('chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,' - 'aes128-gcm@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr') - ciphers_66 = {'default': default, - 'weak': default + ',aes256-cbc,aes192-cbc,aes128-cbc'} - - # Use newer ciphers on ubuntu Trusty and above - _release = lsb_release()['DISTRIB_CODENAME'].lower() - if CompareHostReleases(_release) >= 'trusty': - log('Detected Ubuntu 14.04 or newer, using new ciphers', - level=DEBUG) - cipher = ciphers_66 - - return cipher[weak_ciphers] - - def get_listening(self, listen=['0.0.0.0']): - """Returns a list of addresses SSH can list on - - Turns input into a sensible list of IPs SSH can listen on. Input - must be a python list of interface names, IPs and/or CIDRs. - - :param listen: list of IPs, CIDRs, interface names - - :returns: list of IPs available on the host - """ - if listen == ['0.0.0.0']: - return listen - - value = [] - for network in listen: - try: - ip = get_address_in_network(network=network, fatal=True) - except ValueError: - if is_ip(network): - ip = network - else: - try: - ip = get_iface_addr(iface=network, fatal=False)[0] - except IndexError: - continue - value.append(ip) - if value == []: - return ['0.0.0.0'] - return value - - def __call__(self): - settings = utils.get_settings('ssh') - if settings['common']['network_ipv6_enable']: - addr_family = 'any' - else: - addr_family = 'inet' - - ctxt = { - 'addr_family': addr_family, - 'remote_hosts': settings['common']['remote_hosts'], - 'password_auth_allowed': - settings['client']['password_authentication'], - 'ports': settings['common']['ports'], - 'ciphers': self.get_ciphers(settings['client']['cbc_required']), - 'macs': self.get_macs(settings['client']['weak_hmac']), - 'kexs': self.get_kexs(settings['client']['weak_kex']), - 'roaming': settings['client']['roaming'], - } - return ctxt - - -class SSHConfig(TemplatedFile): - def __init__(self): - path = '/etc/ssh/ssh_config' - super(SSHConfig, self).__init__(path=path, - template_dir=TEMPLATES_DIR, - context=SSHConfigContext(), - user='root', - group='root', - mode=0o0644) - - def pre_write(self): - settings = utils.get_settings('ssh') - apt_update(fatal=True) - apt_install(settings['client']['package']) - if not os.path.exists('/etc/ssh'): - os.makedir('/etc/ssh') - # NOTE: don't recurse - utils.ensure_permissions('/etc/ssh', 'root', 'root', 0o0755, - maxdepth=0) - - def post_write(self): - # NOTE: don't recurse - utils.ensure_permissions('/etc/ssh', 'root', 'root', 0o0755, - maxdepth=0) - - -class SSHDConfigContext(SSHConfigContext): - - type = 'server' - - def __call__(self): - settings = utils.get_settings('ssh') - if settings['common']['network_ipv6_enable']: - addr_family = 'any' - else: - addr_family = 'inet' - - ctxt = { - 'ssh_ip': self.get_listening(settings['server']['listen_to']), - 'password_auth_allowed': - settings['server']['password_authentication'], - 'ports': settings['common']['ports'], - 'addr_family': addr_family, - 'ciphers': self.get_ciphers(settings['server']['cbc_required']), - 'macs': self.get_macs(settings['server']['weak_hmac']), - 'kexs': self.get_kexs(settings['server']['weak_kex']), - 'host_key_files': settings['server']['host_key_files'], - 'allow_root_with_key': settings['server']['allow_root_with_key'], - 'password_authentication': - settings['server']['password_authentication'], - 'use_priv_sep': settings['server']['use_privilege_separation'], - 'use_pam': settings['server']['use_pam'], - 'allow_x11_forwarding': settings['server']['allow_x11_forwarding'], - 'print_motd': settings['server']['print_motd'], - 'print_last_log': settings['server']['print_last_log'], - 'client_alive_interval': - settings['server']['alive_interval'], - 'client_alive_count': settings['server']['alive_count'], - 'allow_tcp_forwarding': settings['server']['allow_tcp_forwarding'], - 'allow_agent_forwarding': - settings['server']['allow_agent_forwarding'], - 'deny_users': settings['server']['deny_users'], - 'allow_users': settings['server']['allow_users'], - 'deny_groups': settings['server']['deny_groups'], - 'allow_groups': settings['server']['allow_groups'], - 'use_dns': settings['server']['use_dns'], - 'sftp_enable': settings['server']['sftp_enable'], - 'sftp_group': settings['server']['sftp_group'], - 'sftp_chroot': settings['server']['sftp_chroot'], - 'max_auth_tries': settings['server']['max_auth_tries'], - 'max_sessions': settings['server']['max_sessions'], - } - return ctxt - - -class SSHDConfig(TemplatedFile): - def __init__(self): - path = '/etc/ssh/sshd_config' - super(SSHDConfig, self).__init__(path=path, - template_dir=TEMPLATES_DIR, - context=SSHDConfigContext(), - user='root', - group='root', - mode=0o0600, - service_actions=[{'service': 'ssh', - 'actions': - ['restart']}]) - - def pre_write(self): - settings = utils.get_settings('ssh') - apt_update(fatal=True) - apt_install(settings['server']['package']) - if not os.path.exists('/etc/ssh'): - os.makedir('/etc/ssh') - # NOTE: don't recurse - utils.ensure_permissions('/etc/ssh', 'root', 'root', 0o0755, - maxdepth=0) - - def post_write(self): - # NOTE: don't recurse - utils.ensure_permissions('/etc/ssh', 'root', 'root', 0o0755, - maxdepth=0) - - -class SSHConfigFileContentAudit(FileContentAudit): - def __init__(self): - self.path = '/etc/ssh/ssh_config' - super(SSHConfigFileContentAudit, self).__init__(self.path, {}) - - def is_compliant(self, *args, **kwargs): - self.pass_cases = [] - self.fail_cases = [] - settings = utils.get_settings('ssh') - - _release = lsb_release()['DISTRIB_CODENAME'].lower() - if CompareHostReleases(_release) >= 'trusty': - if not settings['server']['weak_hmac']: - self.pass_cases.append(r'^MACs.+,hmac-ripemd160$') - else: - self.pass_cases.append(r'^MACs.+,hmac-sha1$') - - if settings['server']['weak_kex']: - self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha256[,\s]?') # noqa - self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group14-sha1[,\s]?') # noqa - self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha1[,\s]?') # noqa - self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group1-sha1[,\s]?') # noqa - else: - self.pass_cases.append(r'^KexAlgorithms.+,diffie-hellman-group-exchange-sha256$') # noqa - self.fail_cases.append(r'^KexAlgorithms.*diffie-hellman-group14-sha1[,\s]?') # noqa - - if settings['server']['cbc_required']: - self.pass_cases.append(r'^Ciphers\s.*-cbc[,\s]?') - self.fail_cases.append(r'^Ciphers\s.*aes128-ctr[,\s]?') - self.fail_cases.append(r'^Ciphers\s.*aes192-ctr[,\s]?') - self.fail_cases.append(r'^Ciphers\s.*aes256-ctr[,\s]?') - else: - self.fail_cases.append(r'^Ciphers\s.*-cbc[,\s]?') - self.pass_cases.append(r'^Ciphers\schacha20-poly1305@openssh.com,.+') # noqa - self.pass_cases.append(r'^Ciphers\s.*aes128-ctr$') - self.pass_cases.append(r'^Ciphers\s.*aes192-ctr[,\s]?') - self.pass_cases.append(r'^Ciphers\s.*aes256-ctr[,\s]?') - else: - if not settings['client']['weak_hmac']: - self.fail_cases.append(r'^MACs.+,hmac-sha1$') - else: - self.pass_cases.append(r'^MACs.+,hmac-sha1$') - - if settings['client']['weak_kex']: - self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha256[,\s]?') # noqa - self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group14-sha1[,\s]?') # noqa - self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha1[,\s]?') # noqa - self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group1-sha1[,\s]?') # noqa - else: - self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha256$') # noqa - self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group14-sha1[,\s]?') # noqa - self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha1[,\s]?') # noqa - self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group1-sha1[,\s]?') # noqa - - if settings['client']['cbc_required']: - self.pass_cases.append(r'^Ciphers\s.*-cbc[,\s]?') - self.fail_cases.append(r'^Ciphers\s.*aes128-ctr[,\s]?') - self.fail_cases.append(r'^Ciphers\s.*aes192-ctr[,\s]?') - self.fail_cases.append(r'^Ciphers\s.*aes256-ctr[,\s]?') - else: - self.fail_cases.append(r'^Ciphers\s.*-cbc[,\s]?') - self.pass_cases.append(r'^Ciphers\s.*aes128-ctr[,\s]?') - self.pass_cases.append(r'^Ciphers\s.*aes192-ctr[,\s]?') - self.pass_cases.append(r'^Ciphers\s.*aes256-ctr[,\s]?') - - if settings['client']['roaming']: - self.pass_cases.append(r'^UseRoaming yes$') - else: - self.fail_cases.append(r'^UseRoaming yes$') - - return super(SSHConfigFileContentAudit, self).is_compliant(*args, - **kwargs) - - -class SSHDConfigFileContentAudit(FileContentAudit): - def __init__(self): - self.path = '/etc/ssh/sshd_config' - super(SSHDConfigFileContentAudit, self).__init__(self.path, {}) - - def is_compliant(self, *args, **kwargs): - self.pass_cases = [] - self.fail_cases = [] - settings = utils.get_settings('ssh') - - _release = lsb_release()['DISTRIB_CODENAME'].lower() - if CompareHostReleases(_release) >= 'trusty': - if not settings['server']['weak_hmac']: - self.pass_cases.append(r'^MACs.+,hmac-ripemd160$') - else: - self.pass_cases.append(r'^MACs.+,hmac-sha1$') - - if settings['server']['weak_kex']: - self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha256[,\s]?') # noqa - self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group14-sha1[,\s]?') # noqa - self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha1[,\s]?') # noqa - self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group1-sha1[,\s]?') # noqa - else: - self.pass_cases.append(r'^KexAlgorithms.+,diffie-hellman-group-exchange-sha256$') # noqa - self.fail_cases.append(r'^KexAlgorithms.*diffie-hellman-group14-sha1[,\s]?') # noqa - - if settings['server']['cbc_required']: - self.pass_cases.append(r'^Ciphers\s.*-cbc[,\s]?') - self.fail_cases.append(r'^Ciphers\s.*aes128-ctr[,\s]?') - self.fail_cases.append(r'^Ciphers\s.*aes192-ctr[,\s]?') - self.fail_cases.append(r'^Ciphers\s.*aes256-ctr[,\s]?') - else: - self.fail_cases.append(r'^Ciphers\s.*-cbc[,\s]?') - self.pass_cases.append(r'^Ciphers\schacha20-poly1305@openssh.com,.+') # noqa - self.pass_cases.append(r'^Ciphers\s.*aes128-ctr$') - self.pass_cases.append(r'^Ciphers\s.*aes192-ctr[,\s]?') - self.pass_cases.append(r'^Ciphers\s.*aes256-ctr[,\s]?') - else: - if not settings['server']['weak_hmac']: - self.pass_cases.append(r'^MACs.+,hmac-ripemd160$') - else: - self.pass_cases.append(r'^MACs.+,hmac-sha1$') - - if settings['server']['weak_kex']: - self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha256[,\s]?') # noqa - self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group14-sha1[,\s]?') # noqa - self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha1[,\s]?') # noqa - self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group1-sha1[,\s]?') # noqa - else: - self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha256$') # noqa - self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group14-sha1[,\s]?') # noqa - self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha1[,\s]?') # noqa - self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group1-sha1[,\s]?') # noqa - - if settings['server']['cbc_required']: - self.pass_cases.append(r'^Ciphers\s.*-cbc[,\s]?') - self.fail_cases.append(r'^Ciphers\s.*aes128-ctr[,\s]?') - self.fail_cases.append(r'^Ciphers\s.*aes192-ctr[,\s]?') - self.fail_cases.append(r'^Ciphers\s.*aes256-ctr[,\s]?') - else: - self.fail_cases.append(r'^Ciphers\s.*-cbc[,\s]?') - self.pass_cases.append(r'^Ciphers\s.*aes128-ctr[,\s]?') - self.pass_cases.append(r'^Ciphers\s.*aes192-ctr[,\s]?') - self.pass_cases.append(r'^Ciphers\s.*aes256-ctr[,\s]?') - - if settings['server']['sftp_enable']: - self.pass_cases.append(r'^Subsystem\ssftp') - else: - self.fail_cases.append(r'^Subsystem\ssftp') - - return super(SSHDConfigFileContentAudit, self).is_compliant(*args, - **kwargs) diff --git a/hooks/charmhelpers/contrib/hardening/ssh/templates/__init__.py b/hooks/charmhelpers/contrib/hardening/ssh/templates/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/hooks/charmhelpers/contrib/hardening/ssh/templates/ssh_config b/hooks/charmhelpers/contrib/hardening/ssh/templates/ssh_config deleted file mode 100644 index 9742d8e..0000000 --- a/hooks/charmhelpers/contrib/hardening/ssh/templates/ssh_config +++ /dev/null @@ -1,70 +0,0 @@ -############################################################################### -# WARNING: This configuration file is maintained by Juju. Local changes may -# be overwritten. -############################################################################### -# This is the ssh client system-wide configuration file. See -# ssh_config(5) for more information. This file provides defaults for -# users, and the values can be changed in per-user configuration files -# or on the command line. - -# Configuration data is parsed as follows: -# 1. command line options -# 2. user-specific file -# 3. system-wide file -# Any configuration value is only changed the first time it is set. -# Thus, host-specific definitions should be at the beginning of the -# configuration file, and defaults at the end. - -# Site-wide defaults for some commonly used options. For a comprehensive -# list of available options, their meanings and defaults, please see the -# ssh_config(5) man page. - -# Restrict the following configuration to be limited to this Host. -{% if remote_hosts -%} -Host {{ ' '.join(remote_hosts) }} -{% endif %} -ForwardAgent no -ForwardX11 no -ForwardX11Trusted yes -RhostsRSAAuthentication no -RSAAuthentication yes -PasswordAuthentication {{ password_auth_allowed }} -HostbasedAuthentication no -GSSAPIAuthentication no -GSSAPIDelegateCredentials no -GSSAPIKeyExchange no -GSSAPITrustDNS no -BatchMode no -CheckHostIP yes -AddressFamily {{ addr_family }} -ConnectTimeout 0 -StrictHostKeyChecking ask -IdentityFile ~/.ssh/identity -IdentityFile ~/.ssh/id_rsa -IdentityFile ~/.ssh/id_dsa -# The port at the destination should be defined -{% for port in ports -%} -Port {{ port }} -{% endfor %} -Protocol 2 -Cipher 3des -{% if ciphers -%} -Ciphers {{ ciphers }} -{%- endif %} -{% if macs -%} -MACs {{ macs }} -{%- endif %} -{% if kexs -%} -KexAlgorithms {{ kexs }} -{%- endif %} -EscapeChar ~ -Tunnel no -TunnelDevice any:any -PermitLocalCommand no -VisualHostKey no -RekeyLimit 1G 1h -SendEnv LANG LC_* -HashKnownHosts yes -{% if roaming -%} -UseRoaming {{ roaming }} -{% endif %} diff --git a/hooks/charmhelpers/contrib/hardening/ssh/templates/sshd_config b/hooks/charmhelpers/contrib/hardening/ssh/templates/sshd_config deleted file mode 100644 index 5f87298..0000000 --- a/hooks/charmhelpers/contrib/hardening/ssh/templates/sshd_config +++ /dev/null @@ -1,159 +0,0 @@ -############################################################################### -# WARNING: This configuration file is maintained by Juju. Local changes may -# be overwritten. -############################################################################### -# Package generated configuration file -# See the sshd_config(5) manpage for details - -# What ports, IPs and protocols we listen for -{% for port in ports -%} -Port {{ port }} -{% endfor -%} -AddressFamily {{ addr_family }} -# Use these options to restrict which interfaces/protocols sshd will bind to -{% if ssh_ip -%} -{% for ip in ssh_ip -%} -ListenAddress {{ ip }} -{% endfor %} -{%- else -%} -ListenAddress :: -ListenAddress 0.0.0.0 -{% endif -%} -Protocol 2 -{% if ciphers -%} -Ciphers {{ ciphers }} -{% endif -%} -{% if macs -%} -MACs {{ macs }} -{% endif -%} -{% if kexs -%} -KexAlgorithms {{ kexs }} -{% endif -%} -# HostKeys for protocol version 2 -{% for keyfile in host_key_files -%} -HostKey {{ keyfile }} -{% endfor -%} - -# Privilege Separation is turned on for security -{% if use_priv_sep -%} -UsePrivilegeSeparation {{ use_priv_sep }} -{% endif -%} - -# Lifetime and size of ephemeral version 1 server key -KeyRegenerationInterval 3600 -ServerKeyBits 1024 - -# Logging -SyslogFacility AUTH -LogLevel VERBOSE - -# Authentication: -LoginGraceTime 30s -{% if allow_root_with_key -%} -PermitRootLogin without-password -{% else -%} -PermitRootLogin no -{% endif %} -PermitTunnel no -PermitUserEnvironment no -StrictModes yes - -RSAAuthentication yes -PubkeyAuthentication yes -AuthorizedKeysFile %h/.ssh/authorized_keys - -# Don't read the user's ~/.rhosts and ~/.shosts files -IgnoreRhosts yes -# For this to work you will also need host keys in /etc/ssh_known_hosts -RhostsRSAAuthentication no -# similar for protocol version 2 -HostbasedAuthentication no -# Uncomment if you don't trust ~/.ssh/known_hosts for RhostsRSAAuthentication -IgnoreUserKnownHosts yes - -# To enable empty passwords, change to yes (NOT RECOMMENDED) -PermitEmptyPasswords no - -# Change to yes to enable challenge-response passwords (beware issues with -# some PAM modules and threads) -ChallengeResponseAuthentication no - -# Change to no to disable tunnelled clear text passwords -PasswordAuthentication {{ password_authentication }} - -# Kerberos options -KerberosAuthentication no -KerberosGetAFSToken no -KerberosOrLocalPasswd no -KerberosTicketCleanup yes - -# GSSAPI options -GSSAPIAuthentication no -GSSAPICleanupCredentials yes - -X11Forwarding {{ allow_x11_forwarding }} -X11DisplayOffset 10 -X11UseLocalhost yes -GatewayPorts no -PrintMotd {{ print_motd }} -PrintLastLog {{ print_last_log }} -TCPKeepAlive no -UseLogin no - -ClientAliveInterval {{ client_alive_interval }} -ClientAliveCountMax {{ client_alive_count }} -AllowTcpForwarding {{ allow_tcp_forwarding }} -AllowAgentForwarding {{ allow_agent_forwarding }} - -MaxStartups 10:30:100 -#Banner /etc/issue.net - -# Allow client to pass locale environment variables -AcceptEnv LANG LC_* - -# Set this to 'yes' to enable PAM authentication, account processing, -# and session processing. If this is enabled, PAM authentication will -# be allowed through the ChallengeResponseAuthentication and -# PasswordAuthentication. Depending on your PAM configuration, -# PAM authentication via ChallengeResponseAuthentication may bypass -# the setting of "PermitRootLogin without-password". -# If you just want the PAM account and session checks to run without -# PAM authentication, then enable this but set PasswordAuthentication -# and ChallengeResponseAuthentication to 'no'. -UsePAM {{ use_pam }} - -{% if deny_users -%} -DenyUsers {{ deny_users }} -{% endif -%} -{% if allow_users -%} -AllowUsers {{ allow_users }} -{% endif -%} -{% if deny_groups -%} -DenyGroups {{ deny_groups }} -{% endif -%} -{% if allow_groups -%} -AllowGroups allow_groups -{% endif -%} -UseDNS {{ use_dns }} -MaxAuthTries {{ max_auth_tries }} -MaxSessions {{ max_sessions }} - -{% if sftp_enable -%} -# Configuration, in case SFTP is used -## override default of no subsystems -## Subsystem sftp /opt/app/openssh5/libexec/sftp-server -Subsystem sftp internal-sftp -l VERBOSE - -## These lines must appear at the *end* of sshd_config -Match Group {{ sftp_group }} -ForceCommand internal-sftp -l VERBOSE -ChrootDirectory {{ sftp_chroot }} -{% else -%} -# Configuration, in case SFTP is used -## override default of no subsystems -## Subsystem sftp /opt/app/openssh5/libexec/sftp-server -## These lines must appear at the *end* of sshd_config -Match Group sftponly -ForceCommand internal-sftp -l VERBOSE -ChrootDirectory /sftpchroot/home/%u -{% endif %} diff --git a/hooks/charmhelpers/contrib/hardening/templating.py b/hooks/charmhelpers/contrib/hardening/templating.py deleted file mode 100644 index 5b6765f..0000000 --- a/hooks/charmhelpers/contrib/hardening/templating.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright 2016 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import six - -from charmhelpers.core.hookenv import ( - log, - DEBUG, - WARNING, -) - -try: - from jinja2 import FileSystemLoader, Environment -except ImportError: - from charmhelpers.fetch import apt_install - from charmhelpers.fetch import apt_update - apt_update(fatal=True) - if six.PY2: - apt_install('python-jinja2', fatal=True) - else: - apt_install('python3-jinja2', fatal=True) - from jinja2 import FileSystemLoader, Environment - - -# NOTE: function separated from main rendering code to facilitate easier -# mocking in unit tests. -def write(path, data): - with open(path, 'wb') as out: - out.write(data) - - -def get_template_path(template_dir, path): - """Returns the template file which would be used to render the path. - - The path to the template file is returned. - :param template_dir: the directory the templates are located in - :param path: the file path to be written to. - :returns: path to the template file - """ - return os.path.join(template_dir, os.path.basename(path)) - - -def render_and_write(template_dir, path, context): - """Renders the specified template into the file. - - :param template_dir: the directory to load the template from - :param path: the path to write the templated contents to - :param context: the parameters to pass to the rendering engine - """ - env = Environment(loader=FileSystemLoader(template_dir)) - template_file = os.path.basename(path) - template = env.get_template(template_file) - log('Rendering from template: %s' % template.name, level=DEBUG) - rendered_content = template.render(context) - if not rendered_content: - log("Render returned None - skipping '%s'" % path, - level=WARNING) - return - - write(path, rendered_content.encode('utf-8').strip()) - log('Wrote template %s' % path, level=DEBUG) diff --git a/hooks/charmhelpers/contrib/hardening/utils.py b/hooks/charmhelpers/contrib/hardening/utils.py deleted file mode 100644 index ff7485c..0000000 --- a/hooks/charmhelpers/contrib/hardening/utils.py +++ /dev/null @@ -1,155 +0,0 @@ -# Copyright 2016 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import glob -import grp -import os -import pwd -import six -import yaml - -from charmhelpers.core.hookenv import ( - log, - DEBUG, - INFO, - WARNING, - ERROR, -) - - -# Global settings cache. Since each hook fire entails a fresh module import it -# is safe to hold this in memory and not risk missing config changes (since -# they will result in a new hook fire and thus re-import). -__SETTINGS__ = {} - - -def _get_defaults(modules): - """Load the default config for the provided modules. - - :param modules: stack modules config defaults to lookup. - :returns: modules default config dictionary. - """ - default = os.path.join(os.path.dirname(__file__), - 'defaults/%s.yaml' % (modules)) - return yaml.safe_load(open(default)) - - -def _get_schema(modules): - """Load the config schema for the provided modules. - - NOTE: this schema is intended to have 1-1 relationship with they keys in - the default config and is used a means to verify valid overrides provided - by the user. - - :param modules: stack modules config schema to lookup. - :returns: modules default schema dictionary. - """ - schema = os.path.join(os.path.dirname(__file__), - 'defaults/%s.yaml.schema' % (modules)) - return yaml.safe_load(open(schema)) - - -def _get_user_provided_overrides(modules): - """Load user-provided config overrides. - - :param modules: stack modules to lookup in user overrides yaml file. - :returns: overrides dictionary. - """ - overrides = os.path.join(os.environ['JUJU_CHARM_DIR'], - 'hardening.yaml') - if os.path.exists(overrides): - log("Found user-provided config overrides file '%s'" % - (overrides), level=DEBUG) - settings = yaml.safe_load(open(overrides)) - if settings and settings.get(modules): - log("Applying '%s' overrides" % (modules), level=DEBUG) - return settings.get(modules) - - log("No overrides found for '%s'" % (modules), level=DEBUG) - else: - log("No hardening config overrides file '%s' found in charm " - "root dir" % (overrides), level=DEBUG) - - return {} - - -def _apply_overrides(settings, overrides, schema): - """Get overrides config overlayed onto modules defaults. - - :param modules: require stack modules config. - :returns: dictionary of modules config with user overrides applied. - """ - if overrides: - for k, v in six.iteritems(overrides): - if k in schema: - if schema[k] is None: - settings[k] = v - elif type(schema[k]) is dict: - settings[k] = _apply_overrides(settings[k], overrides[k], - schema[k]) - else: - raise Exception("Unexpected type found in schema '%s'" % - type(schema[k]), level=ERROR) - else: - log("Unknown override key '%s' - ignoring" % (k), level=INFO) - - return settings - - -def get_settings(modules): - global __SETTINGS__ - if modules in __SETTINGS__: - return __SETTINGS__[modules] - - schema = _get_schema(modules) - settings = _get_defaults(modules) - overrides = _get_user_provided_overrides(modules) - __SETTINGS__[modules] = _apply_overrides(settings, overrides, schema) - return __SETTINGS__[modules] - - -def ensure_permissions(path, user, group, permissions, maxdepth=-1): - """Ensure permissions for path. - - If path is a file, apply to file and return. If path is a directory, - apply recursively (if required) to directory contents and return. - - :param user: user name - :param group: group name - :param permissions: octal permissions - :param maxdepth: maximum recursion depth. A negative maxdepth allows - infinite recursion and maxdepth=0 means no recursion. - :returns: None - """ - if not os.path.exists(path): - log("File '%s' does not exist - cannot set permissions" % (path), - level=WARNING) - return - - _user = pwd.getpwnam(user) - os.chown(path, _user.pw_uid, grp.getgrnam(group).gr_gid) - os.chmod(path, permissions) - - if maxdepth == 0: - log("Max recursion depth reached - skipping further recursion", - level=DEBUG) - return - elif maxdepth > 0: - maxdepth -= 1 - - if os.path.isdir(path): - contents = glob.glob("%s/*" % (path)) - for c in contents: - ensure_permissions(c, user=user, group=group, - permissions=permissions, maxdepth=maxdepth) diff --git a/hooks/charmhelpers/contrib/network/__init__.py b/hooks/charmhelpers/contrib/network/__init__.py deleted file mode 100644 index d7567b8..0000000 --- a/hooks/charmhelpers/contrib/network/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py deleted file mode 100644 index a871ce3..0000000 --- a/hooks/charmhelpers/contrib/network/ip.py +++ /dev/null @@ -1,593 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import glob -import re -import subprocess -import six -import socket - -from functools import partial - -from charmhelpers.fetch import apt_install, apt_update -from charmhelpers.core.hookenv import ( - config, - log, - network_get_primary_address, - unit_get, - WARNING, -) - -from charmhelpers.core.host import ( - lsb_release, - CompareHostReleases, -) - -try: - import netifaces -except ImportError: - apt_update(fatal=True) - if six.PY2: - apt_install('python-netifaces', fatal=True) - else: - apt_install('python3-netifaces', fatal=True) - import netifaces - -try: - import netaddr -except ImportError: - apt_update(fatal=True) - if six.PY2: - apt_install('python-netaddr', fatal=True) - else: - apt_install('python3-netaddr', fatal=True) - import netaddr - - -def _validate_cidr(network): - try: - netaddr.IPNetwork(network) - except (netaddr.core.AddrFormatError, ValueError): - raise ValueError("Network (%s) is not in CIDR presentation format" % - network) - - -def no_ip_found_error_out(network): - errmsg = ("No IP address found in network(s): %s" % network) - raise ValueError(errmsg) - - -def _get_ipv6_network_from_address(address): - """Get an netaddr.IPNetwork for the given IPv6 address - :param address: a dict as returned by netifaces.ifaddresses - :returns netaddr.IPNetwork: None if the address is a link local or loopback - address - """ - if address['addr'].startswith('fe80') or address['addr'] == "::1": - return None - - prefix = address['netmask'].split("/") - if len(prefix) > 1: - netmask = prefix[1] - else: - netmask = address['netmask'] - return netaddr.IPNetwork("%s/%s" % (address['addr'], - netmask)) - - -def get_address_in_network(network, fallback=None, fatal=False): - """Get an IPv4 or IPv6 address within the network from the host. - - :param network (str): CIDR presentation format. For example, - '192.168.1.0/24'. Supports multiple networks as a space-delimited list. - :param fallback (str): If no address is found, return fallback. - :param fatal (boolean): If no address is found, fallback is not - set and fatal is True then exit(1). - """ - if network is None: - if fallback is not None: - return fallback - - if fatal: - no_ip_found_error_out(network) - else: - return None - - networks = network.split() or [network] - for network in networks: - _validate_cidr(network) - network = netaddr.IPNetwork(network) - for iface in netifaces.interfaces(): - addresses = netifaces.ifaddresses(iface) - if network.version == 4 and netifaces.AF_INET in addresses: - for addr in addresses[netifaces.AF_INET]: - cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'], - addr['netmask'])) - if cidr in network: - return str(cidr.ip) - - if network.version == 6 and netifaces.AF_INET6 in addresses: - for addr in addresses[netifaces.AF_INET6]: - cidr = _get_ipv6_network_from_address(addr) - if cidr and cidr in network: - return str(cidr.ip) - - if fallback is not None: - return fallback - - if fatal: - no_ip_found_error_out(network) - - return None - - -def is_ipv6(address): - """Determine whether provided address is IPv6 or not.""" - try: - address = netaddr.IPAddress(address) - except netaddr.AddrFormatError: - # probably a hostname - so not an address at all! - return False - - return address.version == 6 - - -def is_address_in_network(network, address): - """ - Determine whether the provided address is within a network range. - - :param network (str): CIDR presentation format. For example, - '192.168.1.0/24'. - :param address: An individual IPv4 or IPv6 address without a net - mask or subnet prefix. For example, '192.168.1.1'. - :returns boolean: Flag indicating whether address is in network. - """ - try: - network = netaddr.IPNetwork(network) - except (netaddr.core.AddrFormatError, ValueError): - raise ValueError("Network (%s) is not in CIDR presentation format" % - network) - - try: - address = netaddr.IPAddress(address) - except (netaddr.core.AddrFormatError, ValueError): - raise ValueError("Address (%s) is not in correct presentation format" % - address) - - if address in network: - return True - else: - return False - - -def _get_for_address(address, key): - """Retrieve an attribute of or the physical interface that - the IP address provided could be bound to. - - :param address (str): An individual IPv4 or IPv6 address without a net - mask or subnet prefix. For example, '192.168.1.1'. - :param key: 'iface' for the physical interface name or an attribute - of the configured interface, for example 'netmask'. - :returns str: Requested attribute or None if address is not bindable. - """ - address = netaddr.IPAddress(address) - for iface in netifaces.interfaces(): - addresses = netifaces.ifaddresses(iface) - if address.version == 4 and netifaces.AF_INET in addresses: - addr = addresses[netifaces.AF_INET][0]['addr'] - netmask = addresses[netifaces.AF_INET][0]['netmask'] - network = netaddr.IPNetwork("%s/%s" % (addr, netmask)) - cidr = network.cidr - if address in cidr: - if key == 'iface': - return iface - else: - return addresses[netifaces.AF_INET][0][key] - - if address.version == 6 and netifaces.AF_INET6 in addresses: - for addr in addresses[netifaces.AF_INET6]: - network = _get_ipv6_network_from_address(addr) - if not network: - continue - - cidr = network.cidr - if address in cidr: - if key == 'iface': - return iface - elif key == 'netmask' and cidr: - return str(cidr).split('/')[1] - else: - return addr[key] - return None - - -get_iface_for_address = partial(_get_for_address, key='iface') - - -get_netmask_for_address = partial(_get_for_address, key='netmask') - - -def resolve_network_cidr(ip_address): - ''' - Resolves the full address cidr of an ip_address based on - configured network interfaces - ''' - netmask = get_netmask_for_address(ip_address) - return str(netaddr.IPNetwork("%s/%s" % (ip_address, netmask)).cidr) - - -def format_ipv6_addr(address): - """If address is IPv6, wrap it in '[]' otherwise return None. - - This is required by most configuration files when specifying IPv6 - addresses. - """ - if is_ipv6(address): - return "[%s]" % address - - return None - - -def is_ipv6_disabled(): - try: - result = subprocess.check_output( - ['sysctl', 'net.ipv6.conf.all.disable_ipv6'], - stderr=subprocess.STDOUT, - universal_newlines=True) - except subprocess.CalledProcessError: - return True - - return "net.ipv6.conf.all.disable_ipv6 = 1" in result - - -def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False, - fatal=True, exc_list=None): - """Return the assigned IP address for a given interface, if any. - - :param iface: network interface on which address(es) are expected to - be found. - :param inet_type: inet address family - :param inc_aliases: include alias interfaces in search - :param fatal: if True, raise exception if address not found - :param exc_list: list of addresses to ignore - :return: list of ip addresses - """ - # Extract nic if passed /dev/ethX - if '/' in iface: - iface = iface.split('/')[-1] - - if not exc_list: - exc_list = [] - - try: - inet_num = getattr(netifaces, inet_type) - except AttributeError: - raise Exception("Unknown inet type '%s'" % str(inet_type)) - - interfaces = netifaces.interfaces() - if inc_aliases: - ifaces = [] - for _iface in interfaces: - if iface == _iface or _iface.split(':')[0] == iface: - ifaces.append(_iface) - - if fatal and not ifaces: - raise Exception("Invalid interface '%s'" % iface) - - ifaces.sort() - else: - if iface not in interfaces: - if fatal: - raise Exception("Interface '%s' not found " % (iface)) - else: - return [] - - else: - ifaces = [iface] - - addresses = [] - for netiface in ifaces: - net_info = netifaces.ifaddresses(netiface) - if inet_num in net_info: - for entry in net_info[inet_num]: - if 'addr' in entry and entry['addr'] not in exc_list: - addresses.append(entry['addr']) - - if fatal and not addresses: - raise Exception("Interface '%s' doesn't have any %s addresses." % - (iface, inet_type)) - - return sorted(addresses) - - -get_ipv4_addr = partial(get_iface_addr, inet_type='AF_INET') - - -def get_iface_from_addr(addr): - """Work out on which interface the provided address is configured.""" - for iface in netifaces.interfaces(): - addresses = netifaces.ifaddresses(iface) - for inet_type in addresses: - for _addr in addresses[inet_type]: - _addr = _addr['addr'] - # link local - ll_key = re.compile("(.+)%.*") - raw = re.match(ll_key, _addr) - if raw: - _addr = raw.group(1) - - if _addr == addr: - log("Address '%s' is configured on iface '%s'" % - (addr, iface)) - return iface - - msg = "Unable to infer net iface on which '%s' is configured" % (addr) - raise Exception(msg) - - -def sniff_iface(f): - """Ensure decorated function is called with a value for iface. - - If no iface provided, inject net iface inferred from unit private address. - """ - def iface_sniffer(*args, **kwargs): - if not kwargs.get('iface', None): - kwargs['iface'] = get_iface_from_addr(unit_get('private-address')) - - return f(*args, **kwargs) - - return iface_sniffer - - -@sniff_iface -def get_ipv6_addr(iface=None, inc_aliases=False, fatal=True, exc_list=None, - dynamic_only=True): - """Get assigned IPv6 address for a given interface. - - Returns list of addresses found. If no address found, returns empty list. - - If iface is None, we infer the current primary interface by doing a reverse - lookup on the unit private-address. - - We currently only support scope global IPv6 addresses i.e. non-temporary - addresses. If no global IPv6 address is found, return the first one found - in the ipv6 address list. - - :param iface: network interface on which ipv6 address(es) are expected to - be found. - :param inc_aliases: include alias interfaces in search - :param fatal: if True, raise exception if address not found - :param exc_list: list of addresses to ignore - :param dynamic_only: only recognise dynamic addresses - :return: list of ipv6 addresses - """ - addresses = get_iface_addr(iface=iface, inet_type='AF_INET6', - inc_aliases=inc_aliases, fatal=fatal, - exc_list=exc_list) - - if addresses: - global_addrs = [] - for addr in addresses: - key_scope_link_local = re.compile("^fe80::..(.+)%(.+)") - m = re.match(key_scope_link_local, addr) - if m: - eui_64_mac = m.group(1) - iface = m.group(2) - else: - global_addrs.append(addr) - - if global_addrs: - # Make sure any found global addresses are not temporary - cmd = ['ip', 'addr', 'show', iface] - out = subprocess.check_output(cmd).decode('UTF-8') - if dynamic_only: - key = re.compile("inet6 (.+)/[0-9]+ scope global.* dynamic.*") - else: - key = re.compile("inet6 (.+)/[0-9]+ scope global.*") - - addrs = [] - for line in out.split('\n'): - line = line.strip() - m = re.match(key, line) - if m and 'temporary' not in line: - # Return the first valid address we find - for addr in global_addrs: - if m.group(1) == addr: - if not dynamic_only or \ - m.group(1).endswith(eui_64_mac): - addrs.append(addr) - - if addrs: - return addrs - - if fatal: - raise Exception("Interface '%s' does not have a scope global " - "non-temporary ipv6 address." % iface) - - return [] - - -def get_bridges(vnic_dir='/sys/devices/virtual/net'): - """Return a list of bridges on the system.""" - b_regex = "%s/*/bridge" % vnic_dir - return [x.replace(vnic_dir, '').split('/')[1] for x in glob.glob(b_regex)] - - -def get_bridge_nics(bridge, vnic_dir='/sys/devices/virtual/net'): - """Return a list of nics comprising a given bridge on the system.""" - brif_regex = "%s/%s/brif/*" % (vnic_dir, bridge) - return [x.split('/')[-1] for x in glob.glob(brif_regex)] - - -def is_bridge_member(nic): - """Check if a given nic is a member of a bridge.""" - for bridge in get_bridges(): - if nic in get_bridge_nics(bridge): - return True - - return False - - -def is_ip(address): - """ - Returns True if address is a valid IP address. - """ - try: - # Test to see if already an IPv4/IPv6 address - address = netaddr.IPAddress(address) - return True - except (netaddr.AddrFormatError, ValueError): - return False - - -def ns_query(address): - try: - import dns.resolver - except ImportError: - if six.PY2: - apt_install('python-dnspython', fatal=True) - else: - apt_install('python3-dnspython', fatal=True) - import dns.resolver - - if isinstance(address, dns.name.Name): - rtype = 'PTR' - elif isinstance(address, six.string_types): - rtype = 'A' - else: - return None - - try: - answers = dns.resolver.query(address, rtype) - except dns.resolver.NXDOMAIN: - return None - - if answers: - return str(answers[0]) - return None - - -def get_host_ip(hostname, fallback=None): - """ - Resolves the IP for a given hostname, or returns - the input if it is already an IP. - """ - if is_ip(hostname): - return hostname - - ip_addr = ns_query(hostname) - if not ip_addr: - try: - ip_addr = socket.gethostbyname(hostname) - except Exception: - log("Failed to resolve hostname '%s'" % (hostname), - level=WARNING) - return fallback - return ip_addr - - -def get_hostname(address, fqdn=True): - """ - Resolves hostname for given IP, or returns the input - if it is already a hostname. - """ - if is_ip(address): - try: - import dns.reversename - except ImportError: - if six.PY2: - apt_install("python-dnspython", fatal=True) - else: - apt_install("python3-dnspython", fatal=True) - import dns.reversename - - rev = dns.reversename.from_address(address) - result = ns_query(rev) - - if not result: - try: - result = socket.gethostbyaddr(address)[0] - except Exception: - return None - else: - result = address - - if fqdn: - # strip trailing . - if result.endswith('.'): - return result[:-1] - else: - return result - else: - return result.split('.')[0] - - -def port_has_listener(address, port): - """ - Returns True if the address:port is open and being listened to, - else False. - - @param address: an IP address or hostname - @param port: integer port - - Note calls 'zc' via a subprocess shell - """ - cmd = ['nc', '-z', address, str(port)] - result = subprocess.call(cmd) - return not(bool(result)) - - -def assert_charm_supports_ipv6(): - """Check whether we are able to support charms ipv6.""" - release = lsb_release()['DISTRIB_CODENAME'].lower() - if CompareHostReleases(release) < "trusty": - raise Exception("IPv6 is not supported in the charms for Ubuntu " - "versions less than Trusty 14.04") - - -def get_relation_ip(interface, cidr_network=None): - """Return this unit's IP for the given interface. - - Allow for an arbitrary interface to use with network-get to select an IP. - Handle all address selection options including passed cidr network and - IPv6. - - Usage: get_relation_ip('amqp', cidr_network='10.0.0.0/8') - - @param interface: string name of the relation. - @param cidr_network: string CIDR Network to select an address from. - @raises Exception if prefer-ipv6 is configured but IPv6 unsupported. - @returns IPv6 or IPv4 address - """ - # Select the interface address first - # For possible use as a fallback bellow with get_address_in_network - try: - # Get the interface specific IP - address = network_get_primary_address(interface) - except NotImplementedError: - # If network-get is not available - address = get_host_ip(unit_get('private-address')) - - if config('prefer-ipv6'): - # Currently IPv6 has priority, eventually we want IPv6 to just be - # another network space. - assert_charm_supports_ipv6() - return get_ipv6_addr()[0] - elif cidr_network: - # If a specific CIDR network is passed get the address from that - # network. - return get_address_in_network(cidr_network, address) - - # Return the interface address - return address diff --git a/hooks/charmhelpers/contrib/openstack/__init__.py b/hooks/charmhelpers/contrib/openstack/__init__.py deleted file mode 100644 index d7567b8..0000000 --- a/hooks/charmhelpers/contrib/openstack/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/hooks/charmhelpers/contrib/openstack/alternatives.py b/hooks/charmhelpers/contrib/openstack/alternatives.py deleted file mode 100644 index 547de09..0000000 --- a/hooks/charmhelpers/contrib/openstack/alternatives.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -''' Helper for managing alternatives for file conflict resolution ''' - -import subprocess -import shutil -import os - - -def install_alternative(name, target, source, priority=50): - ''' Install alternative configuration ''' - if (os.path.exists(target) and not os.path.islink(target)): - # Move existing file/directory away before installing - shutil.move(target, '{}.bak'.format(target)) - cmd = [ - 'update-alternatives', '--force', '--install', - target, name, source, str(priority) - ] - subprocess.check_call(cmd) - - -def remove_alternative(name, source): - """Remove an installed alternative configuration file - - :param name: string name of the alternative to remove - :param source: string full path to alternative to remove - """ - cmd = [ - 'update-alternatives', '--remove', - name, source - ] - subprocess.check_call(cmd) diff --git a/hooks/charmhelpers/contrib/openstack/exceptions.py b/hooks/charmhelpers/contrib/openstack/exceptions.py deleted file mode 100644 index f85ae4f..0000000 --- a/hooks/charmhelpers/contrib/openstack/exceptions.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -class OSContextError(Exception): - """Raised when an error occurs during context generation. - - This exception is principally used in contrib.openstack.context - """ - pass diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py deleted file mode 100644 index 9e5af34..0000000 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ /dev/null @@ -1,2125 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Common python helper functions used for OpenStack charms. -from collections import OrderedDict -from functools import wraps - -import subprocess -import json -import os -import sys -import re -import itertools -import functools -import shutil - -import six -import traceback -import uuid -import yaml - -from charmhelpers import deprecate - -from charmhelpers.contrib.network import ip - -from charmhelpers.core import unitdata - -from charmhelpers.core.hookenv import ( - action_fail, - action_set, - config, - log as juju_log, - charm_dir, - INFO, - ERROR, - related_units, - relation_ids, - relation_set, - service_name, - status_set, - hook_name, - application_version_set, - cached, -) - -from charmhelpers.core.strutils import BasicStringComparator - -from charmhelpers.contrib.storage.linux.lvm import ( - deactivate_lvm_volume_group, - is_lvm_physical_volume, - remove_lvm_physical_volume, -) - -from charmhelpers.contrib.network.ip import ( - get_ipv6_addr, - is_ipv6, - port_has_listener, -) - -from charmhelpers.contrib.python.packages import ( - pip_create_virtualenv, - pip_install, -) - -from charmhelpers.core.host import ( - lsb_release, - mounts, - umount, - service_running, - service_pause, - service_resume, - restart_on_change_helper, -) -from charmhelpers.fetch import ( - apt_cache, - install_remote, - import_key as fetch_import_key, - add_source as fetch_add_source, - SourceConfigError, - GPGKeyError, - get_upstream_version -) - -from charmhelpers.fetch.snap import ( - snap_install, - snap_refresh, - valid_snap_channel, -) - -from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk -from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device -from charmhelpers.contrib.openstack.exceptions import OSContextError - -CLOUD_ARCHIVE_URL = "http://ubuntu-cloud.archive.canonical.com/ubuntu" -CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA' - -DISTRO_PROPOSED = ('deb http://archive.ubuntu.com/ubuntu/ %s-proposed ' - 'restricted main multiverse universe') - -OPENSTACK_RELEASES = ( - 'diablo', - 'essex', - 'folsom', - 'grizzly', - 'havana', - 'icehouse', - 'juno', - 'kilo', - 'liberty', - 'mitaka', - 'newton', - 'ocata', - 'pike', - 'queens', - 'rocky', -) - -UBUNTU_OPENSTACK_RELEASE = OrderedDict([ - ('oneiric', 'diablo'), - ('precise', 'essex'), - ('quantal', 'folsom'), - ('raring', 'grizzly'), - ('saucy', 'havana'), - ('trusty', 'icehouse'), - ('utopic', 'juno'), - ('vivid', 'kilo'), - ('wily', 'liberty'), - ('xenial', 'mitaka'), - ('yakkety', 'newton'), - ('zesty', 'ocata'), - ('artful', 'pike'), - ('bionic', 'queens'), -]) - - -OPENSTACK_CODENAMES = OrderedDict([ - ('2011.2', 'diablo'), - ('2012.1', 'essex'), - ('2012.2', 'folsom'), - ('2013.1', 'grizzly'), - ('2013.2', 'havana'), - ('2014.1', 'icehouse'), - ('2014.2', 'juno'), - ('2015.1', 'kilo'), - ('2015.2', 'liberty'), - ('2016.1', 'mitaka'), - ('2016.2', 'newton'), - ('2017.1', 'ocata'), - ('2017.2', 'pike'), - ('2018.1', 'queens'), -]) - -# The ugly duckling - must list releases oldest to newest -SWIFT_CODENAMES = OrderedDict([ - ('diablo', - ['1.4.3']), - ('essex', - ['1.4.8']), - ('folsom', - ['1.7.4']), - ('grizzly', - ['1.7.6', '1.7.7', '1.8.0']), - ('havana', - ['1.9.0', '1.9.1', '1.10.0']), - ('icehouse', - ['1.11.0', '1.12.0', '1.13.0', '1.13.1']), - ('juno', - ['2.0.0', '2.1.0', '2.2.0']), - ('kilo', - ['2.2.1', '2.2.2']), - ('liberty', - ['2.3.0', '2.4.0', '2.5.0']), - ('mitaka', - ['2.5.0', '2.6.0', '2.7.0']), - ('newton', - ['2.8.0', '2.9.0', '2.10.0']), - ('ocata', - ['2.11.0', '2.12.0', '2.13.0']), - ('pike', - ['2.13.0', '2.15.0']), - ('queens', - ['2.16.0']), -]) - -# >= Liberty version->codename mapping -PACKAGE_CODENAMES = { - 'nova-common': OrderedDict([ - ('12', 'liberty'), - ('13', 'mitaka'), - ('14', 'newton'), - ('15', 'ocata'), - ('16', 'pike'), - ('17', 'queens'), - ('18', 'rocky'), - ]), - 'neutron-common': OrderedDict([ - ('7', 'liberty'), - ('8', 'mitaka'), - ('9', 'newton'), - ('10', 'ocata'), - ('11', 'pike'), - ('12', 'queens'), - ('13', 'rocky'), - ]), - 'cinder-common': OrderedDict([ - ('7', 'liberty'), - ('8', 'mitaka'), - ('9', 'newton'), - ('10', 'ocata'), - ('11', 'pike'), - ('12', 'queens'), - ('13', 'rocky'), - ]), - 'keystone': OrderedDict([ - ('8', 'liberty'), - ('9', 'mitaka'), - ('10', 'newton'), - ('11', 'ocata'), - ('12', 'pike'), - ('13', 'queens'), - ('14', 'rocky'), - ]), - 'horizon-common': OrderedDict([ - ('8', 'liberty'), - ('9', 'mitaka'), - ('10', 'newton'), - ('11', 'ocata'), - ('12', 'pike'), - ('13', 'queens'), - ('14', 'rocky'), - ]), - 'ceilometer-common': OrderedDict([ - ('5', 'liberty'), - ('6', 'mitaka'), - ('7', 'newton'), - ('8', 'ocata'), - ('9', 'pike'), - ('10', 'queens'), - ('11', 'rocky'), - ]), - 'heat-common': OrderedDict([ - ('5', 'liberty'), - ('6', 'mitaka'), - ('7', 'newton'), - ('8', 'ocata'), - ('9', 'pike'), - ('10', 'queens'), - ('11', 'rocky'), - ]), - 'glance-common': OrderedDict([ - ('11', 'liberty'), - ('12', 'mitaka'), - ('13', 'newton'), - ('14', 'ocata'), - ('15', 'pike'), - ('16', 'queens'), - ('17', 'rocky'), - ]), - 'openstack-dashboard': OrderedDict([ - ('8', 'liberty'), - ('9', 'mitaka'), - ('10', 'newton'), - ('11', 'ocata'), - ('12', 'pike'), - ('13', 'queens'), - ('14', 'rocky'), - ]), -} - -GIT_DEFAULT_REPOS = { - 'requirements': 'git://github.com/openstack/requirements', - 'cinder': 'git://github.com/openstack/cinder', - 'glance': 'git://github.com/openstack/glance', - 'horizon': 'git://github.com/openstack/horizon', - 'keystone': 'git://github.com/openstack/keystone', - 'networking-hyperv': 'git://github.com/openstack/networking-hyperv', - 'neutron': 'git://github.com/openstack/neutron', - 'neutron-fwaas': 'git://github.com/openstack/neutron-fwaas', - 'neutron-lbaas': 'git://github.com/openstack/neutron-lbaas', - 'neutron-vpnaas': 'git://github.com/openstack/neutron-vpnaas', - 'nova': 'git://github.com/openstack/nova', -} - -GIT_DEFAULT_BRANCHES = { - 'liberty': 'stable/liberty', - 'mitaka': 'stable/mitaka', - 'newton': 'stable/newton', - 'master': 'master', -} - -DEFAULT_LOOPBACK_SIZE = '5G' - - -class CompareOpenStackReleases(BasicStringComparator): - """Provide comparisons of OpenStack releases. - - Use in the form of - - if CompareOpenStackReleases(release) > 'mitaka': - # do something with mitaka - """ - _list = OPENSTACK_RELEASES - - -def error_out(msg): - juju_log("FATAL ERROR: %s" % msg, level='ERROR') - sys.exit(1) - - -def get_os_codename_install_source(src): - '''Derive OpenStack release codename from a given installation source.''' - ubuntu_rel = lsb_release()['DISTRIB_CODENAME'] - rel = '' - if src is None: - return rel - if src in ['distro', 'distro-proposed']: - try: - rel = UBUNTU_OPENSTACK_RELEASE[ubuntu_rel] - except KeyError: - e = 'Could not derive openstack release for '\ - 'this Ubuntu release: %s' % ubuntu_rel - error_out(e) - return rel - - if src.startswith('cloud:'): - ca_rel = src.split(':')[1] - ca_rel = ca_rel.split('%s-' % ubuntu_rel)[1].split('/')[0] - return ca_rel - - # Best guess match based on deb string provided - if (src.startswith('deb') or - src.startswith('ppa') or - src.startswith('snap')): - for v in OPENSTACK_CODENAMES.values(): - if v in src: - return v - - -def get_os_version_install_source(src): - codename = get_os_codename_install_source(src) - return get_os_version_codename(codename) - - -def get_os_codename_version(vers): - '''Determine OpenStack codename from version number.''' - try: - return OPENSTACK_CODENAMES[vers] - except KeyError: - e = 'Could not determine OpenStack codename for version %s' % vers - error_out(e) - - -def get_os_version_codename(codename, version_map=OPENSTACK_CODENAMES): - '''Determine OpenStack version number from codename.''' - for k, v in six.iteritems(version_map): - if v == codename: - return k - e = 'Could not derive OpenStack version for '\ - 'codename: %s' % codename - error_out(e) - - -def get_os_version_codename_swift(codename): - '''Determine OpenStack version number of swift from codename.''' - for k, v in six.iteritems(SWIFT_CODENAMES): - if k == codename: - return v[-1] - e = 'Could not derive swift version for '\ - 'codename: %s' % codename - error_out(e) - - -def get_swift_codename(version): - '''Determine OpenStack codename that corresponds to swift version.''' - codenames = [k for k, v in six.iteritems(SWIFT_CODENAMES) if version in v] - - if len(codenames) > 1: - # If more than one release codename contains this version we determine - # the actual codename based on the highest available install source. - for codename in reversed(codenames): - releases = UBUNTU_OPENSTACK_RELEASE - release = [k for k, v in six.iteritems(releases) if codename in v] - ret = subprocess.check_output(['apt-cache', 'policy', 'swift']) - if six.PY3: - ret = ret.decode('UTF-8') - if codename in ret or release[0] in ret: - return codename - elif len(codenames) == 1: - return codenames[0] - - # NOTE: fallback - attempt to match with just major.minor version - match = re.match('^(\d+)\.(\d+)', version) - if match: - major_minor_version = match.group(0) - for codename, versions in six.iteritems(SWIFT_CODENAMES): - for release_version in versions: - if release_version.startswith(major_minor_version): - return codename - - return None - - -def get_os_codename_package(package, fatal=True): - '''Derive OpenStack release codename from an installed package.''' - - if snap_install_requested(): - cmd = ['snap', 'list', package] - try: - out = subprocess.check_output(cmd) - if six.PY3: - out = out.decode('UTF-8') - except subprocess.CalledProcessError as e: - return None - lines = out.split('\n') - for line in lines: - if package in line: - # Second item in list is Version - return line.split()[1] - - import apt_pkg as apt - - cache = apt_cache() - - try: - pkg = cache[package] - except Exception: - if not fatal: - return None - # the package is unknown to the current apt cache. - e = 'Could not determine version of package with no installation '\ - 'candidate: %s' % package - error_out(e) - - if not pkg.current_ver: - if not fatal: - return None - # package is known, but no version is currently installed. - e = 'Could not determine version of uninstalled package: %s' % package - error_out(e) - - vers = apt.upstream_version(pkg.current_ver.ver_str) - if 'swift' in pkg.name: - # Fully x.y.z match for swift versions - match = re.match('^(\d+)\.(\d+)\.(\d+)', vers) - else: - # x.y match only for 20XX.X - # and ignore patch level for other packages - match = re.match('^(\d+)\.(\d+)', vers) - - if match: - vers = match.group(0) - - # Generate a major version number for newer semantic - # versions of openstack projects - major_vers = vers.split('.')[0] - # >= Liberty independent project versions - if (package in PACKAGE_CODENAMES and - major_vers in PACKAGE_CODENAMES[package]): - return PACKAGE_CODENAMES[package][major_vers] - else: - # < Liberty co-ordinated project versions - try: - if 'swift' in pkg.name: - return get_swift_codename(vers) - else: - return OPENSTACK_CODENAMES[vers] - except KeyError: - if not fatal: - return None - e = 'Could not determine OpenStack codename for version %s' % vers - error_out(e) - - -def get_os_version_package(pkg, fatal=True): - '''Derive OpenStack version number from an installed package.''' - codename = get_os_codename_package(pkg, fatal=fatal) - - if not codename: - return None - - if 'swift' in pkg: - vers_map = SWIFT_CODENAMES - for cname, version in six.iteritems(vers_map): - if cname == codename: - return version[-1] - else: - vers_map = OPENSTACK_CODENAMES - for version, cname in six.iteritems(vers_map): - if cname == codename: - return version - # e = "Could not determine OpenStack version for package: %s" % pkg - # error_out(e) - - -# Module local cache variable for the os_release. -_os_rel = None - - -def reset_os_release(): - '''Unset the cached os_release version''' - global _os_rel - _os_rel = None - - -def os_release(package, base='essex', reset_cache=False): - ''' - Returns OpenStack release codename from a cached global. - - If reset_cache then unset the cached os_release version and return the - freshly determined version. - - If the codename can not be determined from either an installed package or - the installation source, the earliest release supported by the charm should - be returned. - ''' - global _os_rel - if reset_cache: - reset_os_release() - if _os_rel: - return _os_rel - _os_rel = ( - git_os_codename_install_source(config('openstack-origin-git')) or - get_os_codename_package(package, fatal=False) or - get_os_codename_install_source(config('openstack-origin')) or - base) - return _os_rel - - -@deprecate("moved to charmhelpers.fetch.import_key()", "2017-07", log=juju_log) -def import_key(keyid): - """Import a key, either ASCII armored, or a GPG key id. - - @param keyid: the key in ASCII armor format, or a GPG key id. - @raises SystemExit() via sys.exit() on failure. - """ - try: - return fetch_import_key(keyid) - except GPGKeyError as e: - error_out("Could not import key: {}".format(str(e))) - - -def get_source_and_pgp_key(source_and_key): - """Look for a pgp key ID or ascii-armor key in the given input. - - :param source_and_key: Sting, "source_spec|keyid" where '|keyid' is - optional. - :returns (source_spec, key_id OR None) as a tuple. Returns None for key_id - if there was no '|' in the source_and_key string. - """ - try: - source, key = source_and_key.split('|', 2) - return source, key or None - except ValueError: - return source_and_key, None - - -@deprecate("use charmhelpers.fetch.add_source() instead.", - "2017-07", log=juju_log) -def configure_installation_source(source_plus_key): - """Configure an installation source. - - The functionality is provided by charmhelpers.fetch.add_source() - The difference between the two functions is that add_source() signature - requires the key to be passed directly, whereas this function passes an - optional key by appending '|' to the end of the source specificiation - 'source'. - - Another difference from add_source() is that the function calls sys.exit(1) - if the configuration fails, whereas add_source() raises - SourceConfigurationError(). Another difference, is that add_source() - silently fails (with a juju_log command) if there is no matching source to - configure, whereas this function fails with a sys.exit(1) - - :param source: String_plus_key -- see above for details. - - Note that the behaviour on error is to log the error to the juju log and - then call sys.exit(1). - """ - if source_plus_key.startswith('snap'): - # Do nothing for snap installs - return - # extract the key if there is one, denoted by a '|' in the rel - source, key = get_source_and_pgp_key(source_plus_key) - - # handle the ordinary sources via add_source - try: - fetch_add_source(source, key, fail_invalid=True) - except SourceConfigError as se: - error_out(str(se)) - - -def config_value_changed(option): - """ - Determine if config value changed since last call to this function. - """ - hook_data = unitdata.HookData() - with hook_data(): - db = unitdata.kv() - current = config(option) - saved = db.get(option) - db.set(option, current) - if saved is None: - return False - return current != saved - - -def save_script_rc(script_path="scripts/scriptrc", **env_vars): - """ - Write an rc file in the charm-delivered directory containing - exported environment variables provided by env_vars. Any charm scripts run - outside the juju hook environment can source this scriptrc to obtain - updated config information necessary to perform health checks or - service changes. - """ - juju_rc_path = "%s/%s" % (charm_dir(), script_path) - if not os.path.exists(os.path.dirname(juju_rc_path)): - os.mkdir(os.path.dirname(juju_rc_path)) - with open(juju_rc_path, 'wt') as rc_script: - rc_script.write( - "#!/bin/bash\n") - [rc_script.write('export %s=%s\n' % (u, p)) - for u, p in six.iteritems(env_vars) if u != "script_path"] - - -def openstack_upgrade_available(package): - """ - Determines if an OpenStack upgrade is available from installation - source, based on version of installed package. - - :param package: str: Name of installed package. - - :returns: bool: : Returns True if configured installation source offers - a newer version of package. - """ - - import apt_pkg as apt - src = config('openstack-origin') - cur_vers = get_os_version_package(package) - if not cur_vers: - # The package has not been installed yet do not attempt upgrade - return False - if "swift" in package: - codename = get_os_codename_install_source(src) - avail_vers = get_os_version_codename_swift(codename) - else: - avail_vers = get_os_version_install_source(src) - apt.init() - if "swift" in package: - major_cur_vers = cur_vers.split('.', 1)[0] - major_avail_vers = avail_vers.split('.', 1)[0] - major_diff = apt.version_compare(major_avail_vers, major_cur_vers) - return avail_vers > cur_vers and (major_diff == 1 or major_diff == 0) - return apt.version_compare(avail_vers, cur_vers) == 1 - - -def ensure_block_device(block_device): - ''' - Confirm block_device, create as loopback if necessary. - - :param block_device: str: Full path of block device to ensure. - - :returns: str: Full path of ensured block device. - ''' - _none = ['None', 'none', None] - if (block_device in _none): - error_out('prepare_storage(): Missing required input: block_device=%s.' - % block_device) - - if block_device.startswith('/dev/'): - bdev = block_device - elif block_device.startswith('/'): - _bd = block_device.split('|') - if len(_bd) == 2: - bdev, size = _bd - else: - bdev = block_device - size = DEFAULT_LOOPBACK_SIZE - bdev = ensure_loopback_device(bdev, size) - else: - bdev = '/dev/%s' % block_device - - if not is_block_device(bdev): - error_out('Failed to locate valid block device at %s' % bdev) - - return bdev - - -def clean_storage(block_device): - ''' - Ensures a block device is clean. That is: - - unmounted - - any lvm volume groups are deactivated - - any lvm physical device signatures removed - - partition table wiped - - :param block_device: str: Full path to block device to clean. - ''' - for mp, d in mounts(): - if d == block_device: - juju_log('clean_storage(): %s is mounted @ %s, unmounting.' % - (d, mp), level=INFO) - umount(mp, persist=True) - - if is_lvm_physical_volume(block_device): - deactivate_lvm_volume_group(block_device) - remove_lvm_physical_volume(block_device) - else: - zap_disk(block_device) - - -is_ip = ip.is_ip -ns_query = ip.ns_query -get_host_ip = ip.get_host_ip -get_hostname = ip.get_hostname - - -def get_matchmaker_map(mm_file='/etc/oslo/matchmaker_ring.json'): - mm_map = {} - if os.path.isfile(mm_file): - with open(mm_file, 'r') as f: - mm_map = json.load(f) - return mm_map - - -def sync_db_with_multi_ipv6_addresses(database, database_user, - relation_prefix=None): - hosts = get_ipv6_addr(dynamic_only=False) - - if config('vip'): - vips = config('vip').split() - for vip in vips: - if vip and is_ipv6(vip): - hosts.append(vip) - - kwargs = {'database': database, - 'username': database_user, - 'hostname': json.dumps(hosts)} - - if relation_prefix: - for key in list(kwargs.keys()): - kwargs["%s_%s" % (relation_prefix, key)] = kwargs[key] - del kwargs[key] - - for rid in relation_ids('shared-db'): - relation_set(relation_id=rid, **kwargs) - - -def os_requires_version(ostack_release, pkg): - """ - Decorator for hook to specify minimum supported release - """ - def wrap(f): - @wraps(f) - def wrapped_f(*args): - if os_release(pkg) < ostack_release: - raise Exception("This hook is not supported on releases" - " before %s" % ostack_release) - f(*args) - return wrapped_f - return wrap - - -def git_install_requested(): - """ - Returns true if openstack-origin-git is specified. - """ - return config('openstack-origin-git') is not None - - -def git_os_codename_install_source(projects_yaml): - """ - Returns OpenStack codename of release being installed from source. - """ - if git_install_requested(): - projects = _git_yaml_load(projects_yaml) - - if projects in GIT_DEFAULT_BRANCHES.keys(): - if projects == 'master': - return 'ocata' - return projects - - if 'release' in projects: - if projects['release'] == 'master': - return 'ocata' - return projects['release'] - - return None - - -def git_default_repos(projects_yaml): - """ - Returns default repos if a default openstack-origin-git value is specified. - """ - service = service_name() - core_project = service - - for default, branch in six.iteritems(GIT_DEFAULT_BRANCHES): - if projects_yaml == default: - - # add the requirements repo first - repo = { - 'name': 'requirements', - 'repository': GIT_DEFAULT_REPOS['requirements'], - 'branch': branch, - } - repos = [repo] - - # neutron-* and nova-* charms require some additional repos - if service in ['neutron-api', 'neutron-gateway', - 'neutron-openvswitch']: - core_project = 'neutron' - if service == 'neutron-api': - repo = { - 'name': 'networking-hyperv', - 'repository': GIT_DEFAULT_REPOS['networking-hyperv'], - 'branch': branch, - } - repos.append(repo) - for project in ['neutron-fwaas', 'neutron-lbaas', - 'neutron-vpnaas', 'nova']: - repo = { - 'name': project, - 'repository': GIT_DEFAULT_REPOS[project], - 'branch': branch, - } - repos.append(repo) - - elif service in ['nova-cloud-controller', 'nova-compute']: - core_project = 'nova' - repo = { - 'name': 'neutron', - 'repository': GIT_DEFAULT_REPOS['neutron'], - 'branch': branch, - } - repos.append(repo) - elif service == 'openstack-dashboard': - core_project = 'horizon' - - # finally add the current service's core project repo - repo = { - 'name': core_project, - 'repository': GIT_DEFAULT_REPOS[core_project], - 'branch': branch, - } - repos.append(repo) - - return yaml.dump(dict(repositories=repos, release=default)) - - return projects_yaml - - -def _git_yaml_load(projects_yaml): - """ - Load the specified yaml into a dictionary. - """ - if not projects_yaml: - return None - - return yaml.load(projects_yaml) - - -requirements_dir = None - - -def git_clone_and_install(projects_yaml, core_project): - """ - Clone/install all specified OpenStack repositories. - - The expected format of projects_yaml is: - - repositories: - - {name: keystone, - repository: 'git://git.openstack.org/openstack/keystone.git', - branch: 'stable/icehouse'} - - {name: requirements, - repository: 'git://git.openstack.org/openstack/requirements.git', - branch: 'stable/icehouse'} - - directory: /mnt/openstack-git - http_proxy: squid-proxy-url - https_proxy: squid-proxy-url - - The directory, http_proxy, and https_proxy keys are optional. - - """ - global requirements_dir - parent_dir = '/mnt/openstack-git' - http_proxy = None - - projects = _git_yaml_load(projects_yaml) - _git_validate_projects_yaml(projects, core_project) - - old_environ = dict(os.environ) - - if 'http_proxy' in projects.keys(): - http_proxy = projects['http_proxy'] - os.environ['http_proxy'] = projects['http_proxy'] - if 'https_proxy' in projects.keys(): - os.environ['https_proxy'] = projects['https_proxy'] - - if 'directory' in projects.keys(): - parent_dir = projects['directory'] - - pip_create_virtualenv(os.path.join(parent_dir, 'venv')) - - # Upgrade setuptools and pip from default virtualenv versions. The default - # versions in trusty break master OpenStack branch deployments. - for p in ['pip', 'setuptools']: - pip_install(p, upgrade=True, proxy=http_proxy, - venv=os.path.join(parent_dir, 'venv')) - - constraints = None - for p in projects['repositories']: - repo = p['repository'] - branch = p['branch'] - depth = '1' - if 'depth' in p.keys(): - depth = p['depth'] - if p['name'] == 'requirements': - repo_dir = _git_clone_and_install_single(repo, branch, depth, - parent_dir, http_proxy, - update_requirements=False) - requirements_dir = repo_dir - constraints = os.path.join(repo_dir, "upper-constraints.txt") - # upper-constraints didn't exist until after icehouse - if not os.path.isfile(constraints): - constraints = None - # use constraints unless project yaml sets use_constraints to false - if 'use_constraints' in projects.keys(): - if not projects['use_constraints']: - constraints = None - else: - repo_dir = _git_clone_and_install_single(repo, branch, depth, - parent_dir, http_proxy, - update_requirements=True, - constraints=constraints) - - os.environ = old_environ - - -def _git_validate_projects_yaml(projects, core_project): - """ - Validate the projects yaml. - """ - _git_ensure_key_exists('repositories', projects) - - for project in projects['repositories']: - _git_ensure_key_exists('name', project.keys()) - _git_ensure_key_exists('repository', project.keys()) - _git_ensure_key_exists('branch', project.keys()) - - if projects['repositories'][0]['name'] != 'requirements': - error_out('{} git repo must be specified first'.format('requirements')) - - if projects['repositories'][-1]['name'] != core_project: - error_out('{} git repo must be specified last'.format(core_project)) - - _git_ensure_key_exists('release', projects) - - -def _git_ensure_key_exists(key, keys): - """ - Ensure that key exists in keys. - """ - if key not in keys: - error_out('openstack-origin-git key \'{}\' is missing'.format(key)) - - -def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy, - update_requirements, constraints=None): - """ - Clone and install a single git repository. - """ - if not os.path.exists(parent_dir): - juju_log('Directory already exists at {}. ' - 'No need to create directory.'.format(parent_dir)) - os.mkdir(parent_dir) - - juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch)) - repo_dir = install_remote( - repo, dest=parent_dir, branch=branch, depth=depth) - - venv = os.path.join(parent_dir, 'venv') - - if update_requirements: - if not requirements_dir: - error_out('requirements repo must be cloned before ' - 'updating from global requirements.') - _git_update_requirements(venv, repo_dir, requirements_dir) - - juju_log('Installing git repo from dir: {}'.format(repo_dir)) - if http_proxy: - pip_install(repo_dir, proxy=http_proxy, venv=venv, - constraints=constraints) - else: - pip_install(repo_dir, venv=venv, constraints=constraints) - - return repo_dir - - -def _git_update_requirements(venv, package_dir, reqs_dir): - """ - Update from global requirements. - - Update an OpenStack git directory's requirements.txt and - test-requirements.txt from global-requirements.txt. - """ - orig_dir = os.getcwd() - os.chdir(reqs_dir) - python = os.path.join(venv, 'bin/python') - cmd = [python, 'update.py', package_dir] - try: - subprocess.check_call(cmd) - except subprocess.CalledProcessError: - package = os.path.basename(package_dir) - error_out("Error updating {} from " - "global-requirements.txt".format(package)) - os.chdir(orig_dir) - - -def git_pip_venv_dir(projects_yaml): - """ - Return the pip virtualenv path. - """ - parent_dir = '/mnt/openstack-git' - - projects = _git_yaml_load(projects_yaml) - - if 'directory' in projects.keys(): - parent_dir = projects['directory'] - - return os.path.join(parent_dir, 'venv') - - -def git_src_dir(projects_yaml, project): - """ - Return the directory where the specified project's source is located. - """ - parent_dir = '/mnt/openstack-git' - - projects = _git_yaml_load(projects_yaml) - - if 'directory' in projects.keys(): - parent_dir = projects['directory'] - - for p in projects['repositories']: - if p['name'] == project: - return os.path.join(parent_dir, os.path.basename(p['repository'])) - - return None - - -def git_yaml_value(projects_yaml, key): - """ - Return the value in projects_yaml for the specified key. - """ - projects = _git_yaml_load(projects_yaml) - - if key in projects.keys(): - return projects[key] - - return None - - -def git_generate_systemd_init_files(templates_dir): - """ - Generate systemd init files. - - Generates and installs systemd init units and script files based on the - *.init.in files contained in the templates_dir directory. - - This code is based on the openstack-pkg-tools package and its init - script generation, which is used by the OpenStack packages. - """ - for f in os.listdir(templates_dir): - # Create the init script and systemd unit file from the template - if f.endswith(".init.in"): - init_in_file = f - init_file = f[:-8] - service_file = "{}.service".format(init_file) - - init_in_source = os.path.join(templates_dir, init_in_file) - init_source = os.path.join(templates_dir, init_file) - service_source = os.path.join(templates_dir, service_file) - - init_dest = os.path.join('/etc/init.d', init_file) - service_dest = os.path.join('/lib/systemd/system', service_file) - - shutil.copyfile(init_in_source, init_source) - with open(init_source, 'a') as outfile: - template = ('/usr/share/openstack-pkg-tools/' - 'init-script-template') - with open(template) as infile: - outfile.write('\n\n{}'.format(infile.read())) - - cmd = ['pkgos-gen-systemd-unit', init_in_source] - subprocess.check_call(cmd) - - if os.path.exists(init_dest): - os.remove(init_dest) - if os.path.exists(service_dest): - os.remove(service_dest) - shutil.copyfile(init_source, init_dest) - shutil.copyfile(service_source, service_dest) - os.chmod(init_dest, 0o755) - - for f in os.listdir(templates_dir): - # If there's a service.in file, use it instead of the generated one - if f.endswith(".service.in"): - service_in_file = f - service_file = f[:-3] - - service_in_source = os.path.join(templates_dir, service_in_file) - service_source = os.path.join(templates_dir, service_file) - service_dest = os.path.join('/lib/systemd/system', service_file) - - shutil.copyfile(service_in_source, service_source) - - if os.path.exists(service_dest): - os.remove(service_dest) - shutil.copyfile(service_source, service_dest) - - for f in os.listdir(templates_dir): - # Generate the systemd unit if there's no existing .service.in - if f.endswith(".init.in"): - init_in_file = f - init_file = f[:-8] - service_in_file = "{}.service.in".format(init_file) - service_file = "{}.service".format(init_file) - - init_in_source = os.path.join(templates_dir, init_in_file) - service_in_source = os.path.join(templates_dir, service_in_file) - service_source = os.path.join(templates_dir, service_file) - service_dest = os.path.join('/lib/systemd/system', service_file) - - if not os.path.exists(service_in_source): - cmd = ['pkgos-gen-systemd-unit', init_in_source] - subprocess.check_call(cmd) - - if os.path.exists(service_dest): - os.remove(service_dest) - shutil.copyfile(service_source, service_dest) - - -def git_determine_usr_bin(): - """Return the /usr/bin path for Apache2 config. - - The /usr/bin path will be located in the virtualenv if the charm - is configured to deploy from source. - """ - if git_install_requested(): - projects_yaml = config('openstack-origin-git') - projects_yaml = git_default_repos(projects_yaml) - return os.path.join(git_pip_venv_dir(projects_yaml), 'bin') - else: - return '/usr/bin' - - -def git_determine_python_path(): - """Return the python-path for Apache2 config. - - Returns 'None' unless the charm is configured to deploy from source, - in which case the path of the virtualenv's site-packages is returned. - """ - if git_install_requested(): - projects_yaml = config('openstack-origin-git') - projects_yaml = git_default_repos(projects_yaml) - return os.path.join(git_pip_venv_dir(projects_yaml), - 'lib/python2.7/site-packages') - else: - return None - - -def os_workload_status(configs, required_interfaces, charm_func=None): - """ - Decorator to set workload status based on complete contexts - """ - def wrap(f): - @wraps(f) - def wrapped_f(*args, **kwargs): - # Run the original function first - f(*args, **kwargs) - # Set workload status now that contexts have been - # acted on - set_os_workload_status(configs, required_interfaces, charm_func) - return wrapped_f - return wrap - - -def set_os_workload_status(configs, required_interfaces, charm_func=None, - services=None, ports=None): - """Set the state of the workload status for the charm. - - This calls _determine_os_workload_status() to get the new state, message - and sets the status using status_set() - - @param configs: a templating.OSConfigRenderer() object - @param required_interfaces: {generic: [specific, specific2, ...]} - @param charm_func: a callable function that returns state, message. The - signature is charm_func(configs) -> (state, message) - @param services: list of strings OR dictionary specifying services/ports - @param ports: OPTIONAL list of port numbers. - @returns state, message: the new workload status, user message - """ - state, message = _determine_os_workload_status( - configs, required_interfaces, charm_func, services, ports) - status_set(state, message) - - -def _determine_os_workload_status( - configs, required_interfaces, charm_func=None, - services=None, ports=None): - """Determine the state of the workload status for the charm. - - This function returns the new workload status for the charm based - on the state of the interfaces, the paused state and whether the - services are actually running and any specified ports are open. - - This checks: - - 1. if the unit should be paused, that it is actually paused. If so the - state is 'maintenance' + message, else 'broken'. - 2. that the interfaces/relations are complete. If they are not then - it sets the state to either 'broken' or 'waiting' and an appropriate - message. - 3. If all the relation data is set, then it checks that the actual - services really are running. If not it sets the state to 'broken'. - - If everything is okay then the state returns 'active'. - - @param configs: a templating.OSConfigRenderer() object - @param required_interfaces: {generic: [specific, specific2, ...]} - @param charm_func: a callable function that returns state, message. The - signature is charm_func(configs) -> (state, message) - @param services: list of strings OR dictionary specifying services/ports - @param ports: OPTIONAL list of port numbers. - @returns state, message: the new workload status, user message - """ - state, message = _ows_check_if_paused(services, ports) - - if state is None: - state, message = _ows_check_generic_interfaces( - configs, required_interfaces) - - if state != 'maintenance' and charm_func: - # _ows_check_charm_func() may modify the state, message - state, message = _ows_check_charm_func( - state, message, lambda: charm_func(configs)) - - if state is None: - state, message = _ows_check_services_running(services, ports) - - if state is None: - state = 'active' - message = "Unit is ready" - juju_log(message, 'INFO') - - return state, message - - -def _ows_check_if_paused(services=None, ports=None): - """Check if the unit is supposed to be paused, and if so check that the - services/ports (if passed) are actually stopped/not being listened to. - - if the unit isn't supposed to be paused, just return None, None - - @param services: OPTIONAL services spec or list of service names. - @param ports: OPTIONAL list of port numbers. - @returns state, message or None, None - """ - if is_unit_paused_set(): - state, message = check_actually_paused(services=services, - ports=ports) - if state is None: - # we're paused okay, so set maintenance and return - state = "maintenance" - message = "Paused. Use 'resume' action to resume normal service." - return state, message - return None, None - - -def _ows_check_generic_interfaces(configs, required_interfaces): - """Check the complete contexts to determine the workload status. - - - Checks for missing or incomplete contexts - - juju log details of missing required data. - - determines the correct workload status - - creates an appropriate message for status_set(...) - - if there are no problems then the function returns None, None - - @param configs: a templating.OSConfigRenderer() object - @params required_interfaces: {generic_interface: [specific_interface], } - @returns state, message or None, None - """ - incomplete_rel_data = incomplete_relation_data(configs, - required_interfaces) - state = None - message = None - missing_relations = set() - incomplete_relations = set() - - for generic_interface, relations_states in incomplete_rel_data.items(): - related_interface = None - missing_data = {} - # Related or not? - for interface, relation_state in relations_states.items(): - if relation_state.get('related'): - related_interface = interface - missing_data = relation_state.get('missing_data') - break - # No relation ID for the generic_interface? - if not related_interface: - juju_log("{} relation is missing and must be related for " - "functionality. ".format(generic_interface), 'WARN') - state = 'blocked' - missing_relations.add(generic_interface) - else: - # Relation ID eists but no related unit - if not missing_data: - # Edge case - relation ID exists but departings - _hook_name = hook_name() - if (('departed' in _hook_name or 'broken' in _hook_name) and - related_interface in _hook_name): - state = 'blocked' - missing_relations.add(generic_interface) - juju_log("{} relation's interface, {}, " - "relationship is departed or broken " - "and is required for functionality." - "".format(generic_interface, related_interface), - "WARN") - # Normal case relation ID exists but no related unit - # (joining) - else: - juju_log("{} relations's interface, {}, is related but has" - " no units in the relation." - "".format(generic_interface, related_interface), - "INFO") - # Related unit exists and data missing on the relation - else: - juju_log("{} relation's interface, {}, is related awaiting " - "the following data from the relationship: {}. " - "".format(generic_interface, related_interface, - ", ".join(missing_data)), "INFO") - if state != 'blocked': - state = 'waiting' - if generic_interface not in missing_relations: - incomplete_relations.add(generic_interface) - - if missing_relations: - message = "Missing relations: {}".format(", ".join(missing_relations)) - if incomplete_relations: - message += "; incomplete relations: {}" \ - "".format(", ".join(incomplete_relations)) - state = 'blocked' - elif incomplete_relations: - message = "Incomplete relations: {}" \ - "".format(", ".join(incomplete_relations)) - state = 'waiting' - - return state, message - - -def _ows_check_charm_func(state, message, charm_func_with_configs): - """Run a custom check function for the charm to see if it wants to - change the state. This is only run if not in 'maintenance' and - tests to see if the new state is more important that the previous - one determined by the interfaces/relations check. - - @param state: the previously determined state so far. - @param message: the user orientated message so far. - @param charm_func: a callable function that returns state, message - @returns state, message strings. - """ - if charm_func_with_configs: - charm_state, charm_message = charm_func_with_configs() - if charm_state != 'active' and charm_state != 'unknown': - state = workload_state_compare(state, charm_state) - if message: - charm_message = charm_message.replace("Incomplete relations: ", - "") - message = "{}, {}".format(message, charm_message) - else: - message = charm_message - return state, message - - -def _ows_check_services_running(services, ports): - """Check that the services that should be running are actually running - and that any ports specified are being listened to. - - @param services: list of strings OR dictionary specifying services/ports - @param ports: list of ports - @returns state, message: strings or None, None - """ - messages = [] - state = None - if services is not None: - services = _extract_services_list_helper(services) - services_running, running = _check_running_services(services) - if not all(running): - messages.append( - "Services not running that should be: {}" - .format(", ".join(_filter_tuples(services_running, False)))) - state = 'blocked' - # also verify that the ports that should be open are open - # NB, that ServiceManager objects only OPTIONALLY have ports - map_not_open, ports_open = ( - _check_listening_on_services_ports(services)) - if not all(ports_open): - # find which service has missing ports. They are in service - # order which makes it a bit easier. - message_parts = {service: ", ".join([str(v) for v in open_ports]) - for service, open_ports in map_not_open.items()} - message = ", ".join( - ["{}: [{}]".format(s, sp) for s, sp in message_parts.items()]) - messages.append( - "Services with ports not open that should be: {}" - .format(message)) - state = 'blocked' - - if ports is not None: - # and we can also check ports which we don't know the service for - ports_open, ports_open_bools = _check_listening_on_ports_list(ports) - if not all(ports_open_bools): - messages.append( - "Ports which should be open, but are not: {}" - .format(", ".join([str(p) for p, v in ports_open - if not v]))) - state = 'blocked' - - if state is not None: - message = "; ".join(messages) - return state, message - - return None, None - - -def _extract_services_list_helper(services): - """Extract a OrderedDict of {service: [ports]} of the supplied services - for use by the other functions. - - The services object can either be: - - None : no services were passed (an empty dict is returned) - - a list of strings - - A dictionary (optionally OrderedDict) {service_name: {'service': ..}} - - An array of [{'service': service_name, ...}, ...] - - @param services: see above - @returns OrderedDict(service: [ports], ...) - """ - if services is None: - return {} - if isinstance(services, dict): - services = services.values() - # either extract the list of services from the dictionary, or if - # it is a simple string, use that. i.e. works with mixed lists. - _s = OrderedDict() - for s in services: - if isinstance(s, dict) and 'service' in s: - _s[s['service']] = s.get('ports', []) - if isinstance(s, str): - _s[s] = [] - return _s - - -def _check_running_services(services): - """Check that the services dict provided is actually running and provide - a list of (service, boolean) tuples for each service. - - Returns both a zipped list of (service, boolean) and a list of booleans - in the same order as the services. - - @param services: OrderedDict of strings: [ports], one for each service to - check. - @returns [(service, boolean), ...], : results for checks - [boolean] : just the result of the service checks - """ - services_running = [service_running(s) for s in services] - return list(zip(services, services_running)), services_running - - -def _check_listening_on_services_ports(services, test=False): - """Check that the unit is actually listening (has the port open) on the - ports that the service specifies are open. If test is True then the - function returns the services with ports that are open rather than - closed. - - Returns an OrderedDict of service: ports and a list of booleans - - @param services: OrderedDict(service: [port, ...], ...) - @param test: default=False, if False, test for closed, otherwise open. - @returns OrderedDict(service: [port-not-open, ...]...), [boolean] - """ - test = not(not(test)) # ensure test is True or False - all_ports = list(itertools.chain(*services.values())) - ports_states = [port_has_listener('0.0.0.0', p) for p in all_ports] - map_ports = OrderedDict() - matched_ports = [p for p, opened in zip(all_ports, ports_states) - if opened == test] # essentially opened xor test - for service, ports in services.items(): - set_ports = set(ports).intersection(matched_ports) - if set_ports: - map_ports[service] = set_ports - return map_ports, ports_states - - -def _check_listening_on_ports_list(ports): - """Check that the ports list given are being listened to - - Returns a list of ports being listened to and a list of the - booleans. - - @param ports: LIST or port numbers. - @returns [(port_num, boolean), ...], [boolean] - """ - ports_open = [port_has_listener('0.0.0.0', p) for p in ports] - return zip(ports, ports_open), ports_open - - -def _filter_tuples(services_states, state): - """Return a simple list from a list of tuples according to the condition - - @param services_states: LIST of (string, boolean): service and running - state. - @param state: Boolean to match the tuple against. - @returns [LIST of strings] that matched the tuple RHS. - """ - return [s for s, b in services_states if b == state] - - -def workload_state_compare(current_workload_state, workload_state): - """ Return highest priority of two states""" - hierarchy = {'unknown': -1, - 'active': 0, - 'maintenance': 1, - 'waiting': 2, - 'blocked': 3, - } - - if hierarchy.get(workload_state) is None: - workload_state = 'unknown' - if hierarchy.get(current_workload_state) is None: - current_workload_state = 'unknown' - - # Set workload_state based on hierarchy of statuses - if hierarchy.get(current_workload_state) > hierarchy.get(workload_state): - return current_workload_state - else: - return workload_state - - -def incomplete_relation_data(configs, required_interfaces): - """Check complete contexts against required_interfaces - Return dictionary of incomplete relation data. - - configs is an OSConfigRenderer object with configs registered - - required_interfaces is a dictionary of required general interfaces - with dictionary values of possible specific interfaces. - Example: - required_interfaces = {'database': ['shared-db', 'pgsql-db']} - - The interface is said to be satisfied if anyone of the interfaces in the - list has a complete context. - - Return dictionary of incomplete or missing required contexts with relation - status of interfaces and any missing data points. Example: - {'message': - {'amqp': {'missing_data': ['rabbitmq_password'], 'related': True}, - 'zeromq-configuration': {'related': False}}, - 'identity': - {'identity-service': {'related': False}}, - 'database': - {'pgsql-db': {'related': False}, - 'shared-db': {'related': True}}} - """ - complete_ctxts = configs.complete_contexts() - incomplete_relations = [ - svc_type - for svc_type, interfaces in required_interfaces.items() - if not set(interfaces).intersection(complete_ctxts)] - return { - i: configs.get_incomplete_context_data(required_interfaces[i]) - for i in incomplete_relations} - - -def do_action_openstack_upgrade(package, upgrade_callback, configs): - """Perform action-managed OpenStack upgrade. - - Upgrades packages to the configured openstack-origin version and sets - the corresponding action status as a result. - - If the charm was installed from source we cannot upgrade it. - For backwards compatibility a config flag (action-managed-upgrade) must - be set for this code to run, otherwise a full service level upgrade will - fire on config-changed. - - @param package: package name for determining if upgrade available - @param upgrade_callback: function callback to charm's upgrade function - @param configs: templating object derived from OSConfigRenderer class - - @return: True if upgrade successful; False if upgrade failed or skipped - """ - ret = False - - if git_install_requested(): - action_set({'outcome': 'installed from source, skipped upgrade.'}) - else: - if openstack_upgrade_available(package): - if config('action-managed-upgrade'): - juju_log('Upgrading OpenStack release') - - try: - upgrade_callback(configs=configs) - action_set({'outcome': 'success, upgrade completed.'}) - ret = True - except Exception: - action_set({'outcome': 'upgrade failed, see traceback.'}) - action_set({'traceback': traceback.format_exc()}) - action_fail('do_openstack_upgrade resulted in an ' - 'unexpected error') - else: - action_set({'outcome': 'action-managed-upgrade config is ' - 'False, skipped upgrade.'}) - else: - action_set({'outcome': 'no upgrade available.'}) - - return ret - - -def remote_restart(rel_name, remote_service=None): - trigger = { - 'restart-trigger': str(uuid.uuid4()), - } - if remote_service: - trigger['remote-service'] = remote_service - for rid in relation_ids(rel_name): - # This subordinate can be related to two seperate services using - # different subordinate relations so only issue the restart if - # the principle is conencted down the relation we think it is - if related_units(relid=rid): - relation_set(relation_id=rid, - relation_settings=trigger, - ) - - -def check_actually_paused(services=None, ports=None): - """Check that services listed in the services object and and ports - are actually closed (not listened to), to verify that the unit is - properly paused. - - @param services: See _extract_services_list_helper - @returns status, : string for status (None if okay) - message : string for problem for status_set - """ - state = None - message = None - messages = [] - if services is not None: - services = _extract_services_list_helper(services) - services_running, services_states = _check_running_services(services) - if any(services_states): - # there shouldn't be any running so this is a problem - messages.append("these services running: {}" - .format(", ".join( - _filter_tuples(services_running, True)))) - state = "blocked" - ports_open, ports_open_bools = ( - _check_listening_on_services_ports(services, True)) - if any(ports_open_bools): - message_parts = {service: ", ".join([str(v) for v in open_ports]) - for service, open_ports in ports_open.items()} - message = ", ".join( - ["{}: [{}]".format(s, sp) for s, sp in message_parts.items()]) - messages.append( - "these service:ports are open: {}".format(message)) - state = 'blocked' - if ports is not None: - ports_open, bools = _check_listening_on_ports_list(ports) - if any(bools): - messages.append( - "these ports which should be closed, but are open: {}" - .format(", ".join([str(p) for p, v in ports_open if v]))) - state = 'blocked' - if messages: - message = ("Services should be paused but {}" - .format(", ".join(messages))) - return state, message - - -def set_unit_paused(): - """Set the unit to a paused state in the local kv() store. - This does NOT actually pause the unit - """ - with unitdata.HookData()() as t: - kv = t[0] - kv.set('unit-paused', True) - - -def clear_unit_paused(): - """Clear the unit from a paused state in the local kv() store - This does NOT actually restart any services - it only clears the - local state. - """ - with unitdata.HookData()() as t: - kv = t[0] - kv.set('unit-paused', False) - - -def is_unit_paused_set(): - """Return the state of the kv().get('unit-paused'). - This does NOT verify that the unit really is paused. - - To help with units that don't have HookData() (testing) - if it excepts, return False - """ - try: - with unitdata.HookData()() as t: - kv = t[0] - # transform something truth-y into a Boolean. - return not(not(kv.get('unit-paused'))) - except Exception: - return False - - -def pause_unit(assess_status_func, services=None, ports=None, - charm_func=None): - """Pause a unit by stopping the services and setting 'unit-paused' - in the local kv() store. - - Also checks that the services have stopped and ports are no longer - being listened to. - - An optional charm_func() can be called that can either raise an - Exception or return non None, None to indicate that the unit - didn't pause cleanly. - - The signature for charm_func is: - charm_func() -> message: string - - charm_func() is executed after any services are stopped, if supplied. - - The services object can either be: - - None : no services were passed (an empty dict is returned) - - a list of strings - - A dictionary (optionally OrderedDict) {service_name: {'service': ..}} - - An array of [{'service': service_name, ...}, ...] - - @param assess_status_func: (f() -> message: string | None) or None - @param services: OPTIONAL see above - @param ports: OPTIONAL list of port - @param charm_func: function to run for custom charm pausing. - @returns None - @raises Exception(message) on an error for action_fail(). - """ - services = _extract_services_list_helper(services) - messages = [] - if services: - for service in services.keys(): - stopped = service_pause(service) - if not stopped: - messages.append("{} didn't stop cleanly.".format(service)) - if charm_func: - try: - message = charm_func() - if message: - messages.append(message) - except Exception as e: - message.append(str(e)) - set_unit_paused() - if assess_status_func: - message = assess_status_func() - if message: - messages.append(message) - if messages: - raise Exception("Couldn't pause: {}".format("; ".join(messages))) - - -def resume_unit(assess_status_func, services=None, ports=None, - charm_func=None): - """Resume a unit by starting the services and clearning 'unit-paused' - in the local kv() store. - - Also checks that the services have started and ports are being listened to. - - An optional charm_func() can be called that can either raise an - Exception or return non None to indicate that the unit - didn't resume cleanly. - - The signature for charm_func is: - charm_func() -> message: string - - charm_func() is executed after any services are started, if supplied. - - The services object can either be: - - None : no services were passed (an empty dict is returned) - - a list of strings - - A dictionary (optionally OrderedDict) {service_name: {'service': ..}} - - An array of [{'service': service_name, ...}, ...] - - @param assess_status_func: (f() -> message: string | None) or None - @param services: OPTIONAL see above - @param ports: OPTIONAL list of port - @param charm_func: function to run for custom charm resuming. - @returns None - @raises Exception(message) on an error for action_fail(). - """ - services = _extract_services_list_helper(services) - messages = [] - if services: - for service in services.keys(): - started = service_resume(service) - if not started: - messages.append("{} didn't start cleanly.".format(service)) - if charm_func: - try: - message = charm_func() - if message: - messages.append(message) - except Exception as e: - message.append(str(e)) - clear_unit_paused() - if assess_status_func: - message = assess_status_func() - if message: - messages.append(message) - if messages: - raise Exception("Couldn't resume: {}".format("; ".join(messages))) - - -def make_assess_status_func(*args, **kwargs): - """Creates an assess_status_func() suitable for handing to pause_unit() - and resume_unit(). - - This uses the _determine_os_workload_status(...) function to determine - what the workload_status should be for the unit. If the unit is - not in maintenance or active states, then the message is returned to - the caller. This is so an action that doesn't result in either a - complete pause or complete resume can signal failure with an action_fail() - """ - def _assess_status_func(): - state, message = _determine_os_workload_status(*args, **kwargs) - status_set(state, message) - if state not in ['maintenance', 'active']: - return message - return None - - return _assess_status_func - - -def pausable_restart_on_change(restart_map, stopstart=False, - restart_functions=None): - """A restart_on_change decorator that checks to see if the unit is - paused. If it is paused then the decorated function doesn't fire. - - This is provided as a helper, as the @restart_on_change(...) decorator - is in core.host, yet the openstack specific helpers are in this file - (contrib.openstack.utils). Thus, this needs to be an optional feature - for openstack charms (or charms that wish to use the openstack - pause/resume type features). - - It is used as follows: - - from contrib.openstack.utils import ( - pausable_restart_on_change as restart_on_change) - - @restart_on_change(restart_map, stopstart=) - def some_hook(...): - pass - - see core.utils.restart_on_change() for more details. - - @param f: the function to decorate - @param restart_map: the restart map {conf_file: [services]} - @param stopstart: DEFAULT false; whether to stop, start or just restart - @returns decorator to use a restart_on_change with pausability - """ - def wrap(f): - @functools.wraps(f) - def wrapped_f(*args, **kwargs): - if is_unit_paused_set(): - return f(*args, **kwargs) - # otherwise, normal restart_on_change functionality - return restart_on_change_helper( - (lambda: f(*args, **kwargs)), restart_map, stopstart, - restart_functions) - return wrapped_f - return wrap - - -def ordered(orderme): - """Converts the provided dictionary into a collections.OrderedDict. - - The items in the returned OrderedDict will be inserted based on the - natural sort order of the keys. Nested dictionaries will also be sorted - in order to ensure fully predictable ordering. - - :param orderme: the dict to order - :return: collections.OrderedDict - :raises: ValueError: if `orderme` isn't a dict instance. - """ - if not isinstance(orderme, dict): - raise ValueError('argument must be a dict type') - - result = OrderedDict() - for k, v in sorted(six.iteritems(orderme), key=lambda x: x[0]): - if isinstance(v, dict): - result[k] = ordered(v) - else: - result[k] = v - - return result - - -def config_flags_parser(config_flags): - """Parses config flags string into dict. - - This parsing method supports a few different formats for the config - flag values to be parsed: - - 1. A string in the simple format of key=value pairs, with the possibility - of specifying multiple key value pairs within the same string. For - example, a string in the format of 'key1=value1, key2=value2' will - return a dict of: - - {'key1': 'value1', 'key2': 'value2'}. - - 2. A string in the above format, but supporting a comma-delimited list - of values for the same key. For example, a string in the format of - 'key1=value1, key2=value3,value4,value5' will return a dict of: - - {'key1': 'value1', 'key2': 'value2,value3,value4'} - - 3. A string containing a colon character (:) prior to an equal - character (=) will be treated as yaml and parsed as such. This can be - used to specify more complex key value pairs. For example, - a string in the format of 'key1: subkey1=value1, subkey2=value2' will - return a dict of: - - {'key1', 'subkey1=value1, subkey2=value2'} - - The provided config_flags string may be a list of comma-separated values - which themselves may be comma-separated list of values. - """ - # If we find a colon before an equals sign then treat it as yaml. - # Note: limit it to finding the colon first since this indicates assignment - # for inline yaml. - colon = config_flags.find(':') - equals = config_flags.find('=') - if colon > 0: - if colon < equals or equals < 0: - return ordered(yaml.safe_load(config_flags)) - - if config_flags.find('==') >= 0: - juju_log("config_flags is not in expected format (key=value)", - level=ERROR) - raise OSContextError - - # strip the following from each value. - post_strippers = ' ,' - # we strip any leading/trailing '=' or ' ' from the string then - # split on '='. - split = config_flags.strip(' =').split('=') - limit = len(split) - flags = OrderedDict() - for i in range(0, limit - 1): - current = split[i] - next = split[i + 1] - vindex = next.rfind(',') - if (i == limit - 2) or (vindex < 0): - value = next - else: - value = next[:vindex] - - if i == 0: - key = current - else: - # if this not the first entry, expect an embedded key. - index = current.rfind(',') - if index < 0: - juju_log("Invalid config value(s) at index %s" % (i), - level=ERROR) - raise OSContextError - key = current[index + 1:] - - # Add to collection. - flags[key.strip(post_strippers)] = value.rstrip(post_strippers) - - return flags - - -def os_application_version_set(package): - '''Set version of application for Juju 2.0 and later''' - application_version = get_upstream_version(package) - # NOTE(jamespage) if not able to figure out package version, fallback to - # openstack codename version detection. - if not application_version: - application_version_set(os_release(package)) - else: - application_version_set(application_version) - - -def enable_memcache(source=None, release=None, package=None): - """Determine if memcache should be enabled on the local unit - - @param release: release of OpenStack currently deployed - @param package: package to derive OpenStack version deployed - @returns boolean Whether memcache should be enabled - """ - _release = None - if release: - _release = release - else: - _release = os_release(package, base='icehouse') - if not _release: - _release = get_os_codename_install_source(source) - - return CompareOpenStackReleases(_release) >= 'mitaka' - - -def token_cache_pkgs(source=None, release=None): - """Determine additional packages needed for token caching - - @param source: source string for charm - @param release: release of OpenStack currently deployed - @returns List of package to enable token caching - """ - packages = [] - if enable_memcache(source=source, release=release): - packages.extend(['memcached', 'python-memcache']) - return packages - - -def update_json_file(filename, items): - """Updates the json `filename` with a given dict. - :param filename: json filename (i.e.: /etc/glance/policy.json) - :param items: dict of items to update - """ - with open(filename) as fd: - policy = json.load(fd) - policy.update(items) - with open(filename, "w") as fd: - fd.write(json.dumps(policy, indent=4)) - - -@cached -def snap_install_requested(): - """ Determine if installing from snaps - - If openstack-origin is of the form snap:track/channel[/branch] - and channel is in SNAPS_CHANNELS return True. - """ - origin = config('openstack-origin') or "" - if not origin.startswith('snap:'): - return False - - _src = origin[5:] - if '/' in _src: - channel = _src.split('/')[1] - else: - # Handle snap:track with no channel - channel = 'stable' - return valid_snap_channel(channel) - - -def get_snaps_install_info_from_origin(snaps, src, mode='classic'): - """Generate a dictionary of snap install information from origin - - @param snaps: List of snaps - @param src: String of openstack-origin or source of the form - snap:track/channel - @param mode: String classic, devmode or jailmode - @returns: Dictionary of snaps with channels and modes - """ - - if not src.startswith('snap:'): - juju_log("Snap source is not a snap origin", 'WARN') - return {} - - _src = src[5:] - channel = '--channel={}'.format(_src) - - return {snap: {'channel': channel, 'mode': mode} - for snap in snaps} - - -def install_os_snaps(snaps, refresh=False): - """Install OpenStack snaps from channel and with mode - - @param snaps: Dictionary of snaps with channels and modes of the form: - {'snap_name': {'channel': 'snap_channel', - 'mode': 'snap_mode'}} - Where channel is a snapstore channel and mode is --classic, --devmode - or --jailmode. - @param post_snap_install: Callback function to run after snaps have been - installed - """ - - def _ensure_flag(flag): - if flag.startswith('--'): - return flag - return '--{}'.format(flag) - - if refresh: - for snap in snaps.keys(): - snap_refresh(snap, - _ensure_flag(snaps[snap]['channel']), - _ensure_flag(snaps[snap]['mode'])) - else: - for snap in snaps.keys(): - snap_install(snap, - _ensure_flag(snaps[snap]['channel']), - _ensure_flag(snaps[snap]['mode'])) diff --git a/hooks/charmhelpers/contrib/python/__init__.py b/hooks/charmhelpers/contrib/python/__init__.py deleted file mode 100644 index d7567b8..0000000 --- a/hooks/charmhelpers/contrib/python/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py deleted file mode 100644 index 6e95028..0000000 --- a/hooks/charmhelpers/contrib/python/packages.py +++ /dev/null @@ -1,154 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import six -import subprocess -import sys - -from charmhelpers.fetch import apt_install, apt_update -from charmhelpers.core.hookenv import charm_dir, log - -__author__ = "Jorge Niedbalski " - - -def pip_execute(*args, **kwargs): - """Overriden pip_execute() to stop sys.path being changed. - - The act of importing main from the pip module seems to cause add wheels - from the /usr/share/python-wheels which are installed by various tools. - This function ensures that sys.path remains the same after the call is - executed. - """ - try: - _path = sys.path - try: - from pip import main as _pip_execute - except ImportError: - apt_update() - if six.PY2: - apt_install('python-pip') - else: - apt_install('python3-pip') - from pip import main as _pip_execute - _pip_execute(*args, **kwargs) - finally: - sys.path = _path - - -def parse_options(given, available): - """Given a set of options, check if available""" - for key, value in sorted(given.items()): - if not value: - continue - if key in available: - yield "--{0}={1}".format(key, value) - - -def pip_install_requirements(requirements, constraints=None, **options): - """Install a requirements file. - - :param constraints: Path to pip constraints file. - http://pip.readthedocs.org/en/stable/user_guide/#constraints-files - """ - command = ["install"] - - available_options = ('proxy', 'src', 'log', ) - for option in parse_options(options, available_options): - command.append(option) - - command.append("-r {0}".format(requirements)) - if constraints: - command.append("-c {0}".format(constraints)) - log("Installing from file: {} with constraints {} " - "and options: {}".format(requirements, constraints, command)) - else: - log("Installing from file: {} with options: {}".format(requirements, - command)) - pip_execute(command) - - -def pip_install(package, fatal=False, upgrade=False, venv=None, - constraints=None, **options): - """Install a python package""" - if venv: - venv_python = os.path.join(venv, 'bin/pip') - command = [venv_python, "install"] - else: - command = ["install"] - - available_options = ('proxy', 'src', 'log', 'index-url', ) - for option in parse_options(options, available_options): - command.append(option) - - if upgrade: - command.append('--upgrade') - - if constraints: - command.extend(['-c', constraints]) - - if isinstance(package, list): - command.extend(package) - else: - command.append(package) - - log("Installing {} package with options: {}".format(package, - command)) - if venv: - subprocess.check_call(command) - else: - pip_execute(command) - - -def pip_uninstall(package, **options): - """Uninstall a python package""" - command = ["uninstall", "-q", "-y"] - - available_options = ('proxy', 'log', ) - for option in parse_options(options, available_options): - command.append(option) - - if isinstance(package, list): - command.extend(package) - else: - command.append(package) - - log("Uninstalling {} package with options: {}".format(package, - command)) - pip_execute(command) - - -def pip_list(): - """Returns the list of current python installed packages - """ - return pip_execute(["list"]) - - -def pip_create_virtualenv(path=None): - """Create an isolated Python environment.""" - if six.PY2: - apt_install('python-virtualenv') - else: - apt_install('python3-virtualenv') - - if path: - venv_path = path - else: - venv_path = os.path.join(charm_dir(), 'venv') - - if not os.path.exists(venv_path): - subprocess.check_call(['virtualenv', venv_path]) diff --git a/hooks/charmhelpers/contrib/storage/__init__.py b/hooks/charmhelpers/contrib/storage/__init__.py deleted file mode 100644 index d7567b8..0000000 --- a/hooks/charmhelpers/contrib/storage/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/hooks/charmhelpers/contrib/storage/linux/__init__.py b/hooks/charmhelpers/contrib/storage/linux/__init__.py deleted file mode 100644 index d7567b8..0000000 --- a/hooks/charmhelpers/contrib/storage/linux/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/hooks/charmhelpers/contrib/storage/linux/bcache.py b/hooks/charmhelpers/contrib/storage/linux/bcache.py deleted file mode 100644 index 605991e..0000000 --- a/hooks/charmhelpers/contrib/storage/linux/bcache.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright 2017 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os -import json - -from charmhelpers.core.hookenv import log - -stats_intervals = ['stats_day', 'stats_five_minute', - 'stats_hour', 'stats_total'] - -SYSFS = '/sys' - - -class Bcache(object): - """Bcache behaviour - """ - - def __init__(self, cachepath): - self.cachepath = cachepath - - @classmethod - def fromdevice(cls, devname): - return cls('{}/block/{}/bcache'.format(SYSFS, devname)) - - def __str__(self): - return self.cachepath - - def get_stats(self, interval): - """Get cache stats - """ - intervaldir = 'stats_{}'.format(interval) - path = "{}/{}".format(self.cachepath, intervaldir) - out = dict() - for elem in os.listdir(path): - out[elem] = open('{}/{}'.format(path, elem)).read().strip() - return out - - -def get_bcache_fs(): - """Return all cache sets - """ - cachesetroot = "{}/fs/bcache".format(SYSFS) - try: - dirs = os.listdir(cachesetroot) - except OSError: - log("No bcache fs found") - return [] - cacheset = set([Bcache('{}/{}'.format(cachesetroot, d)) for d in dirs if not d.startswith('register')]) - return cacheset - - -def get_stats_action(cachespec, interval): - """Action for getting bcache statistics for a given cachespec. - Cachespec can either be a device name, eg. 'sdb', which will retrieve - cache stats for the given device, or 'global', which will retrieve stats - for all cachesets - """ - if cachespec == 'global': - caches = get_bcache_fs() - else: - caches = [Bcache.fromdevice(cachespec)] - res = dict((c.cachepath, c.get_stats(interval)) for c in caches) - return json.dumps(res, indent=4, separators=(',', ': ')) diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py deleted file mode 100644 index 0d9bacf..0000000 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ /dev/null @@ -1,1411 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# -# Copyright 2012 Canonical Ltd. -# -# This file is sourced from lp:openstack-charm-helpers -# -# Authors: -# James Page -# Adam Gandelman -# - -import errno -import hashlib -import math -import six - -import os -import shutil -import json -import time -import uuid - -from subprocess import ( - check_call, - check_output, - CalledProcessError, -) -from charmhelpers.core.hookenv import ( - config, - service_name, - local_unit, - relation_get, - relation_ids, - relation_set, - related_units, - log, - DEBUG, - INFO, - WARNING, - ERROR, -) -from charmhelpers.core.host import ( - mount, - mounts, - service_start, - service_stop, - service_running, - umount, -) -from charmhelpers.fetch import ( - apt_install, -) -from charmhelpers.core.unitdata import kv - -from charmhelpers.core.kernel import modprobe -from charmhelpers.contrib.openstack.utils import config_flags_parser - -KEYRING = '/etc/ceph/ceph.client.{}.keyring' -KEYFILE = '/etc/ceph/ceph.client.{}.key' - -CEPH_CONF = """[global] -auth supported = {auth} -keyring = {keyring} -mon host = {mon_hosts} -log to syslog = {use_syslog} -err to syslog = {use_syslog} -clog to syslog = {use_syslog} -""" - -# The number of placement groups per OSD to target for placement group -# calculations. This number is chosen as 100 due to the ceph PG Calc -# documentation recommending to choose 100 for clusters which are not -# expected to increase in the foreseeable future. Since the majority of the -# calculations are done on deployment, target the case of non-expanding -# clusters as the default. -DEFAULT_PGS_PER_OSD_TARGET = 100 -DEFAULT_POOL_WEIGHT = 10.0 -LEGACY_PG_COUNT = 200 -DEFAULT_MINIMUM_PGS = 2 - - -def validator(value, valid_type, valid_range=None): - """ - Used to validate these: http://docs.ceph.com/docs/master/rados/operations/pools/#set-pool-values - Example input: - validator(value=1, - valid_type=int, - valid_range=[0, 2]) - This says I'm testing value=1. It must be an int inclusive in [0,2] - - :param value: The value to validate - :param valid_type: The type that value should be. - :param valid_range: A range of values that value can assume. - :return: - """ - assert isinstance(value, valid_type), "{} is not a {}".format( - value, - valid_type) - if valid_range is not None: - assert isinstance(valid_range, list), \ - "valid_range must be a list, was given {}".format(valid_range) - # If we're dealing with strings - if valid_type is six.string_types: - assert value in valid_range, \ - "{} is not in the list {}".format(value, valid_range) - # Integer, float should have a min and max - else: - if len(valid_range) != 2: - raise ValueError( - "Invalid valid_range list of {} for {}. " - "List must be [min,max]".format(valid_range, value)) - assert value >= valid_range[0], \ - "{} is less than minimum allowed value of {}".format( - value, valid_range[0]) - assert value <= valid_range[1], \ - "{} is greater than maximum allowed value of {}".format( - value, valid_range[1]) - - -class PoolCreationError(Exception): - """ - A custom error to inform the caller that a pool creation failed. Provides an error message - """ - - def __init__(self, message): - super(PoolCreationError, self).__init__(message) - - -class Pool(object): - """ - An object oriented approach to Ceph pool creation. This base class is inherited by ReplicatedPool and ErasurePool. - Do not call create() on this base class as it will not do anything. Instantiate a child class and call create(). - """ - - def __init__(self, service, name): - self.service = service - self.name = name - - # Create the pool if it doesn't exist already - # To be implemented by subclasses - def create(self): - pass - - def add_cache_tier(self, cache_pool, mode): - """ - Adds a new cache tier to an existing pool. - :param cache_pool: six.string_types. The cache tier pool name to add. - :param mode: six.string_types. The caching mode to use for this pool. valid range = ["readonly", "writeback"] - :return: None - """ - # Check the input types and values - validator(value=cache_pool, valid_type=six.string_types) - validator(value=mode, valid_type=six.string_types, valid_range=["readonly", "writeback"]) - - check_call(['ceph', '--id', self.service, 'osd', 'tier', 'add', self.name, cache_pool]) - check_call(['ceph', '--id', self.service, 'osd', 'tier', 'cache-mode', cache_pool, mode]) - check_call(['ceph', '--id', self.service, 'osd', 'tier', 'set-overlay', self.name, cache_pool]) - check_call(['ceph', '--id', self.service, 'osd', 'pool', 'set', cache_pool, 'hit_set_type', 'bloom']) - - def remove_cache_tier(self, cache_pool): - """ - Removes a cache tier from Ceph. Flushes all dirty objects from writeback pools and waits for that to complete. - :param cache_pool: six.string_types. The cache tier pool name to remove. - :return: None - """ - # read-only is easy, writeback is much harder - mode = get_cache_mode(self.service, cache_pool) - version = ceph_version() - if mode == 'readonly': - check_call(['ceph', '--id', self.service, 'osd', 'tier', 'cache-mode', cache_pool, 'none']) - check_call(['ceph', '--id', self.service, 'osd', 'tier', 'remove', self.name, cache_pool]) - - elif mode == 'writeback': - pool_forward_cmd = ['ceph', '--id', self.service, 'osd', 'tier', - 'cache-mode', cache_pool, 'forward'] - if version >= '10.1': - # Jewel added a mandatory flag - pool_forward_cmd.append('--yes-i-really-mean-it') - - check_call(pool_forward_cmd) - # Flush the cache and wait for it to return - check_call(['rados', '--id', self.service, '-p', cache_pool, 'cache-flush-evict-all']) - check_call(['ceph', '--id', self.service, 'osd', 'tier', 'remove-overlay', self.name]) - check_call(['ceph', '--id', self.service, 'osd', 'tier', 'remove', self.name, cache_pool]) - - def get_pgs(self, pool_size, percent_data=DEFAULT_POOL_WEIGHT): - """Return the number of placement groups to use when creating the pool. - - Returns the number of placement groups which should be specified when - creating the pool. This is based upon the calculation guidelines - provided by the Ceph Placement Group Calculator (located online at - http://ceph.com/pgcalc/). - - The number of placement groups are calculated using the following: - - (Target PGs per OSD) * (OSD #) * (%Data) - ---------------------------------------- - (Pool size) - - Per the upstream guidelines, the OSD # should really be considered - based on the number of OSDs which are eligible to be selected by the - pool. Since the pool creation doesn't specify any of CRUSH set rules, - the default rule will be dependent upon the type of pool being - created (replicated or erasure). - - This code makes no attempt to determine the number of OSDs which can be - selected for the specific rule, rather it is left to the user to tune - in the form of 'expected-osd-count' config option. - - :param pool_size: int. pool_size is either the number of replicas for - replicated pools or the K+M sum for erasure coded pools - :param percent_data: float. the percentage of data that is expected to - be contained in the pool for the specific OSD set. Default value - is to assume 10% of the data is for this pool, which is a - relatively low % of the data but allows for the pg_num to be - increased. NOTE: the default is primarily to handle the scenario - where related charms requiring pools has not been upgraded to - include an update to indicate their relative usage of the pools. - :return: int. The number of pgs to use. - """ - - # Note: This calculation follows the approach that is provided - # by the Ceph PG Calculator located at http://ceph.com/pgcalc/. - validator(value=pool_size, valid_type=int) - - # Ensure that percent data is set to something - even with a default - # it can be set to None, which would wreak havoc below. - if percent_data is None: - percent_data = DEFAULT_POOL_WEIGHT - - # If the expected-osd-count is specified, then use the max between - # the expected-osd-count and the actual osd_count - osd_list = get_osds(self.service) - expected = config('expected-osd-count') or 0 - - if osd_list: - osd_count = max(expected, len(osd_list)) - - # Log a message to provide some insight if the calculations claim - # to be off because someone is setting the expected count and - # there are more OSDs in reality. Try to make a proper guess - # based upon the cluster itself. - if expected and osd_count != expected: - log("Found more OSDs than provided expected count. " - "Using the actual count instead", INFO) - elif expected: - # Use the expected-osd-count in older ceph versions to allow for - # a more accurate pg calculations - osd_count = expected - else: - # NOTE(james-page): Default to 200 for older ceph versions - # which don't support OSD query from cli - return LEGACY_PG_COUNT - - percent_data /= 100.0 - target_pgs_per_osd = config('pgs-per-osd') or DEFAULT_PGS_PER_OSD_TARGET - num_pg = (target_pgs_per_osd * osd_count * percent_data) // pool_size - - # NOTE: ensure a sane minimum number of PGS otherwise we don't get any - # reasonable data distribution in minimal OSD configurations - if num_pg < DEFAULT_MINIMUM_PGS: - num_pg = DEFAULT_MINIMUM_PGS - - # The CRUSH algorithm has a slight optimization for placement groups - # with powers of 2 so find the nearest power of 2. If the nearest - # power of 2 is more than 25% below the original value, the next - # highest value is used. To do this, find the nearest power of 2 such - # that 2^n <= num_pg, check to see if its within the 25% tolerance. - exponent = math.floor(math.log(num_pg, 2)) - nearest = 2 ** exponent - if (num_pg - nearest) > (num_pg * 0.25): - # Choose the next highest power of 2 since the nearest is more - # than 25% below the original value. - return int(nearest * 2) - else: - return int(nearest) - - -class ReplicatedPool(Pool): - def __init__(self, service, name, pg_num=None, replicas=2, - percent_data=10.0): - super(ReplicatedPool, self).__init__(service=service, name=name) - self.replicas = replicas - if pg_num: - # Since the number of placement groups were specified, ensure - # that there aren't too many created. - max_pgs = self.get_pgs(self.replicas, 100.0) - self.pg_num = min(pg_num, max_pgs) - else: - self.pg_num = self.get_pgs(self.replicas, percent_data) - - def create(self): - if not pool_exists(self.service, self.name): - # Create it - cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create', - self.name, str(self.pg_num)] - try: - check_call(cmd) - # Set the pool replica size - update_pool(client=self.service, - pool=self.name, - settings={'size': str(self.replicas)}) - except CalledProcessError: - raise - - -# Default jerasure erasure coded pool -class ErasurePool(Pool): - def __init__(self, service, name, erasure_code_profile="default", - percent_data=10.0): - super(ErasurePool, self).__init__(service=service, name=name) - self.erasure_code_profile = erasure_code_profile - self.percent_data = percent_data - - def create(self): - if not pool_exists(self.service, self.name): - # Try to find the erasure profile information in order to properly - # size the number of placement groups. The size of an erasure - # coded placement group is calculated as k+m. - erasure_profile = get_erasure_profile(self.service, - self.erasure_code_profile) - - # Check for errors - if erasure_profile is None: - msg = ("Failed to discover erasure profile named " - "{}".format(self.erasure_code_profile)) - log(msg, level=ERROR) - raise PoolCreationError(msg) - if 'k' not in erasure_profile or 'm' not in erasure_profile: - # Error - msg = ("Unable to find k (data chunks) or m (coding chunks) " - "in erasure profile {}".format(erasure_profile)) - log(msg, level=ERROR) - raise PoolCreationError(msg) - - k = int(erasure_profile['k']) - m = int(erasure_profile['m']) - pgs = self.get_pgs(k + m, self.percent_data) - # Create it - cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create', - self.name, str(pgs), str(pgs), - 'erasure', self.erasure_code_profile] - try: - check_call(cmd) - except CalledProcessError: - raise - - """Get an existing erasure code profile if it already exists. - Returns json formatted output""" - - -def get_mon_map(service): - """ - Returns the current monitor map. - :param service: six.string_types. The Ceph user name to run the command under - :return: json string. :raise: ValueError if the monmap fails to parse. - Also raises CalledProcessError if our ceph command fails - """ - try: - mon_status = check_output(['ceph', '--id', service, - 'mon_status', '--format=json']) - if six.PY3: - mon_status = mon_status.decode('UTF-8') - try: - return json.loads(mon_status) - except ValueError as v: - log("Unable to parse mon_status json: {}. Error: {}" - .format(mon_status, str(v))) - raise - except CalledProcessError as e: - log("mon_status command failed with message: {}" - .format(str(e))) - raise - - -def hash_monitor_names(service): - """ - Uses the get_mon_map() function to get information about the monitor - cluster. - Hash the name of each monitor. Return a sorted list of monitor hashes - in an ascending order. - :param service: six.string_types. The Ceph user name to run the command under - :rtype : dict. json dict of monitor name, ip address and rank - example: { - 'name': 'ip-172-31-13-165', - 'rank': 0, - 'addr': '172.31.13.165:6789/0'} - """ - try: - hash_list = [] - monitor_list = get_mon_map(service=service) - if monitor_list['monmap']['mons']: - for mon in monitor_list['monmap']['mons']: - hash_list.append( - hashlib.sha224(mon['name'].encode('utf-8')).hexdigest()) - return sorted(hash_list) - else: - return None - except (ValueError, CalledProcessError): - raise - - -def monitor_key_delete(service, key): - """ - Delete a key and value pair from the monitor cluster - :param service: six.string_types. The Ceph user name to run the command under - Deletes a key value pair on the monitor cluster. - :param key: six.string_types. The key to delete. - """ - try: - check_output( - ['ceph', '--id', service, - 'config-key', 'del', str(key)]) - except CalledProcessError as e: - log("Monitor config-key put failed with message: {}".format( - e.output)) - raise - - -def monitor_key_set(service, key, value): - """ - Sets a key value pair on the monitor cluster. - :param service: six.string_types. The Ceph user name to run the command under - :param key: six.string_types. The key to set. - :param value: The value to set. This will be converted to a string - before setting - """ - try: - check_output( - ['ceph', '--id', service, - 'config-key', 'put', str(key), str(value)]) - except CalledProcessError as e: - log("Monitor config-key put failed with message: {}".format( - e.output)) - raise - - -def monitor_key_get(service, key): - """ - Gets the value of an existing key in the monitor cluster. - :param service: six.string_types. The Ceph user name to run the command under - :param key: six.string_types. The key to search for. - :return: Returns the value of that key or None if not found. - """ - try: - output = check_output( - ['ceph', '--id', service, - 'config-key', 'get', str(key)]).decode('UTF-8') - return output - except CalledProcessError as e: - log("Monitor config-key get failed with message: {}".format( - e.output)) - return None - - -def monitor_key_exists(service, key): - """ - Searches for the existence of a key in the monitor cluster. - :param service: six.string_types. The Ceph user name to run the command under - :param key: six.string_types. The key to search for - :return: Returns True if the key exists, False if not and raises an - exception if an unknown error occurs. :raise: CalledProcessError if - an unknown error occurs - """ - try: - check_call( - ['ceph', '--id', service, - 'config-key', 'exists', str(key)]) - # I can return true here regardless because Ceph returns - # ENOENT if the key wasn't found - return True - except CalledProcessError as e: - if e.returncode == errno.ENOENT: - return False - else: - log("Unknown error from ceph config-get exists: {} {}".format( - e.returncode, e.output)) - raise - - -def get_erasure_profile(service, name): - """ - :param service: six.string_types. The Ceph user name to run the command under - :param name: - :return: - """ - try: - out = check_output(['ceph', '--id', service, - 'osd', 'erasure-code-profile', 'get', - name, '--format=json']) - if six.PY3: - out = out.decode('UTF-8') - return json.loads(out) - except (CalledProcessError, OSError, ValueError): - return None - - -def pool_set(service, pool_name, key, value): - """ - Sets a value for a RADOS pool in ceph. - :param service: six.string_types. The Ceph user name to run the command under - :param pool_name: six.string_types - :param key: six.string_types - :param value: - :return: None. Can raise CalledProcessError - """ - cmd = ['ceph', '--id', service, 'osd', 'pool', 'set', pool_name, key, value] - try: - check_call(cmd) - except CalledProcessError: - raise - - -def snapshot_pool(service, pool_name, snapshot_name): - """ - Snapshots a RADOS pool in ceph. - :param service: six.string_types. The Ceph user name to run the command under - :param pool_name: six.string_types - :param snapshot_name: six.string_types - :return: None. Can raise CalledProcessError - """ - cmd = ['ceph', '--id', service, 'osd', 'pool', 'mksnap', pool_name, snapshot_name] - try: - check_call(cmd) - except CalledProcessError: - raise - - -def remove_pool_snapshot(service, pool_name, snapshot_name): - """ - Remove a snapshot from a RADOS pool in ceph. - :param service: six.string_types. The Ceph user name to run the command under - :param pool_name: six.string_types - :param snapshot_name: six.string_types - :return: None. Can raise CalledProcessError - """ - cmd = ['ceph', '--id', service, 'osd', 'pool', 'rmsnap', pool_name, snapshot_name] - try: - check_call(cmd) - except CalledProcessError: - raise - - -# max_bytes should be an int or long -def set_pool_quota(service, pool_name, max_bytes): - """ - :param service: six.string_types. The Ceph user name to run the command under - :param pool_name: six.string_types - :param max_bytes: int or long - :return: None. Can raise CalledProcessError - """ - # Set a byte quota on a RADOS pool in ceph. - cmd = ['ceph', '--id', service, 'osd', 'pool', 'set-quota', pool_name, - 'max_bytes', str(max_bytes)] - try: - check_call(cmd) - except CalledProcessError: - raise - - -def remove_pool_quota(service, pool_name): - """ - Set a byte quota on a RADOS pool in ceph. - :param service: six.string_types. The Ceph user name to run the command under - :param pool_name: six.string_types - :return: None. Can raise CalledProcessError - """ - cmd = ['ceph', '--id', service, 'osd', 'pool', 'set-quota', pool_name, 'max_bytes', '0'] - try: - check_call(cmd) - except CalledProcessError: - raise - - -def remove_erasure_profile(service, profile_name): - """ - Create a new erasure code profile if one does not already exist for it. Updates - the profile if it exists. Please see http://docs.ceph.com/docs/master/rados/operations/erasure-code-profile/ - for more details - :param service: six.string_types. The Ceph user name to run the command under - :param profile_name: six.string_types - :return: None. Can raise CalledProcessError - """ - cmd = ['ceph', '--id', service, 'osd', 'erasure-code-profile', 'rm', - profile_name] - try: - check_call(cmd) - except CalledProcessError: - raise - - -def create_erasure_profile(service, profile_name, erasure_plugin_name='jerasure', - failure_domain='host', - data_chunks=2, coding_chunks=1, - locality=None, durability_estimator=None): - """ - Create a new erasure code profile if one does not already exist for it. Updates - the profile if it exists. Please see http://docs.ceph.com/docs/master/rados/operations/erasure-code-profile/ - for more details - :param service: six.string_types. The Ceph user name to run the command under - :param profile_name: six.string_types - :param erasure_plugin_name: six.string_types - :param failure_domain: six.string_types. One of ['chassis', 'datacenter', 'host', 'osd', 'pdu', 'pod', 'rack', 'region', - 'room', 'root', 'row']) - :param data_chunks: int - :param coding_chunks: int - :param locality: int - :param durability_estimator: int - :return: None. Can raise CalledProcessError - """ - # Ensure this failure_domain is allowed by Ceph - validator(failure_domain, six.string_types, - ['chassis', 'datacenter', 'host', 'osd', 'pdu', 'pod', 'rack', 'region', 'room', 'root', 'row']) - - cmd = ['ceph', '--id', service, 'osd', 'erasure-code-profile', 'set', profile_name, - 'plugin=' + erasure_plugin_name, 'k=' + str(data_chunks), 'm=' + str(coding_chunks), - 'ruleset_failure_domain=' + failure_domain] - if locality is not None and durability_estimator is not None: - raise ValueError("create_erasure_profile should be called with k, m and one of l or c but not both.") - - # Add plugin specific information - if locality is not None: - # For local erasure codes - cmd.append('l=' + str(locality)) - if durability_estimator is not None: - # For Shec erasure codes - cmd.append('c=' + str(durability_estimator)) - - if erasure_profile_exists(service, profile_name): - cmd.append('--force') - - try: - check_call(cmd) - except CalledProcessError: - raise - - -def rename_pool(service, old_name, new_name): - """ - Rename a Ceph pool from old_name to new_name - :param service: six.string_types. The Ceph user name to run the command under - :param old_name: six.string_types - :param new_name: six.string_types - :return: None - """ - validator(value=old_name, valid_type=six.string_types) - validator(value=new_name, valid_type=six.string_types) - - cmd = ['ceph', '--id', service, 'osd', 'pool', 'rename', old_name, new_name] - check_call(cmd) - - -def erasure_profile_exists(service, name): - """ - Check to see if an Erasure code profile already exists. - :param service: six.string_types. The Ceph user name to run the command under - :param name: six.string_types - :return: int or None - """ - validator(value=name, valid_type=six.string_types) - try: - check_call(['ceph', '--id', service, - 'osd', 'erasure-code-profile', 'get', - name]) - return True - except CalledProcessError: - return False - - -def get_cache_mode(service, pool_name): - """ - Find the current caching mode of the pool_name given. - :param service: six.string_types. The Ceph user name to run the command under - :param pool_name: six.string_types - :return: int or None - """ - validator(value=service, valid_type=six.string_types) - validator(value=pool_name, valid_type=six.string_types) - out = check_output(['ceph', '--id', service, - 'osd', 'dump', '--format=json']) - if six.PY3: - out = out.decode('UTF-8') - try: - osd_json = json.loads(out) - for pool in osd_json['pools']: - if pool['pool_name'] == pool_name: - return pool['cache_mode'] - return None - except ValueError: - raise - - -def pool_exists(service, name): - """Check to see if a RADOS pool already exists.""" - try: - out = check_output(['rados', '--id', service, 'lspools']) - if six.PY3: - out = out.decode('UTF-8') - except CalledProcessError: - return False - - return name in out.split() - - -def get_osds(service): - """Return a list of all Ceph Object Storage Daemons currently in the - cluster. - """ - version = ceph_version() - if version and version >= '0.56': - out = check_output(['ceph', '--id', service, - 'osd', 'ls', - '--format=json']) - if six.PY3: - out = out.decode('UTF-8') - return json.loads(out) - - return None - - -def install(): - """Basic Ceph client installation.""" - ceph_dir = "/etc/ceph" - if not os.path.exists(ceph_dir): - os.mkdir(ceph_dir) - - apt_install('ceph-common', fatal=True) - - -def rbd_exists(service, pool, rbd_img): - """Check to see if a RADOS block device exists.""" - try: - out = check_output(['rbd', 'list', '--id', - service, '--pool', pool]) - if six.PY3: - out = out.decode('UTF-8') - except CalledProcessError: - return False - - return rbd_img in out - - -def create_rbd_image(service, pool, image, sizemb): - """Create a new RADOS block device.""" - cmd = ['rbd', 'create', image, '--size', str(sizemb), '--id', service, - '--pool', pool] - check_call(cmd) - - -def update_pool(client, pool, settings): - cmd = ['ceph', '--id', client, 'osd', 'pool', 'set', pool] - for k, v in six.iteritems(settings): - cmd.append(k) - cmd.append(v) - - check_call(cmd) - - -def create_pool(service, name, replicas=3, pg_num=None): - """Create a new RADOS pool.""" - if pool_exists(service, name): - log("Ceph pool {} already exists, skipping creation".format(name), - level=WARNING) - return - - if not pg_num: - # Calculate the number of placement groups based - # on upstream recommended best practices. - osds = get_osds(service) - if osds: - pg_num = (len(osds) * 100 // replicas) - else: - # NOTE(james-page): Default to 200 for older ceph versions - # which don't support OSD query from cli - pg_num = 200 - - cmd = ['ceph', '--id', service, 'osd', 'pool', 'create', name, str(pg_num)] - check_call(cmd) - - update_pool(service, name, settings={'size': str(replicas)}) - - -def delete_pool(service, name): - """Delete a RADOS pool from ceph.""" - cmd = ['ceph', '--id', service, 'osd', 'pool', 'delete', name, - '--yes-i-really-really-mean-it'] - check_call(cmd) - - -def _keyfile_path(service): - return KEYFILE.format(service) - - -def _keyring_path(service): - return KEYRING.format(service) - - -def create_keyring(service, key): - """Create a new Ceph keyring containing key.""" - keyring = _keyring_path(service) - if os.path.exists(keyring): - log('Ceph keyring exists at %s.' % keyring, level=WARNING) - return - - cmd = ['ceph-authtool', keyring, '--create-keyring', - '--name=client.{}'.format(service), '--add-key={}'.format(key)] - check_call(cmd) - log('Created new ceph keyring at %s.' % keyring, level=DEBUG) - - -def delete_keyring(service): - """Delete an existing Ceph keyring.""" - keyring = _keyring_path(service) - if not os.path.exists(keyring): - log('Keyring does not exist at %s' % keyring, level=WARNING) - return - - os.remove(keyring) - log('Deleted ring at %s.' % keyring, level=INFO) - - -def create_key_file(service, key): - """Create a file containing key.""" - keyfile = _keyfile_path(service) - if os.path.exists(keyfile): - log('Keyfile exists at %s.' % keyfile, level=WARNING) - return - - with open(keyfile, 'w') as fd: - fd.write(key) - - log('Created new keyfile at %s.' % keyfile, level=INFO) - - -def get_ceph_nodes(relation='ceph'): - """Query named relation to determine current nodes.""" - hosts = [] - for r_id in relation_ids(relation): - for unit in related_units(r_id): - hosts.append(relation_get('private-address', unit=unit, rid=r_id)) - - return hosts - - -def configure(service, key, auth, use_syslog): - """Perform basic configuration of Ceph.""" - create_keyring(service, key) - create_key_file(service, key) - hosts = get_ceph_nodes() - with open('/etc/ceph/ceph.conf', 'w') as ceph_conf: - ceph_conf.write(CEPH_CONF.format(auth=auth, - keyring=_keyring_path(service), - mon_hosts=",".join(map(str, hosts)), - use_syslog=use_syslog)) - modprobe('rbd') - - -def image_mapped(name): - """Determine whether a RADOS block device is mapped locally.""" - try: - out = check_output(['rbd', 'showmapped']) - if six.PY3: - out = out.decode('UTF-8') - except CalledProcessError: - return False - - return name in out - - -def map_block_storage(service, pool, image): - """Map a RADOS block device for local use.""" - cmd = [ - 'rbd', - 'map', - '{}/{}'.format(pool, image), - '--user', - service, - '--secret', - _keyfile_path(service), - ] - check_call(cmd) - - -def filesystem_mounted(fs): - """Determine whether a filesytems is already mounted.""" - return fs in [f for f, m in mounts()] - - -def make_filesystem(blk_device, fstype='ext4', timeout=10): - """Make a new filesystem on the specified block device.""" - count = 0 - e_noent = os.errno.ENOENT - while not os.path.exists(blk_device): - if count >= timeout: - log('Gave up waiting on block device %s' % blk_device, - level=ERROR) - raise IOError(e_noent, os.strerror(e_noent), blk_device) - - log('Waiting for block device %s to appear' % blk_device, - level=DEBUG) - count += 1 - time.sleep(1) - else: - log('Formatting block device %s as filesystem %s.' % - (blk_device, fstype), level=INFO) - check_call(['mkfs', '-t', fstype, blk_device]) - - -def place_data_on_block_device(blk_device, data_src_dst): - """Migrate data in data_src_dst to blk_device and then remount.""" - # mount block device into /mnt - mount(blk_device, '/mnt') - # copy data to /mnt - copy_files(data_src_dst, '/mnt') - # umount block device - umount('/mnt') - # Grab user/group ID's from original source - _dir = os.stat(data_src_dst) - uid = _dir.st_uid - gid = _dir.st_gid - # re-mount where the data should originally be - # TODO: persist is currently a NO-OP in core.host - mount(blk_device, data_src_dst, persist=True) - # ensure original ownership of new mount. - os.chown(data_src_dst, uid, gid) - - -def copy_files(src, dst, symlinks=False, ignore=None): - """Copy files from src to dst.""" - for item in os.listdir(src): - s = os.path.join(src, item) - d = os.path.join(dst, item) - if os.path.isdir(s): - shutil.copytree(s, d, symlinks, ignore) - else: - shutil.copy2(s, d) - - -def ensure_ceph_storage(service, pool, rbd_img, sizemb, mount_point, - blk_device, fstype, system_services=[], - replicas=3): - """NOTE: This function must only be called from a single service unit for - the same rbd_img otherwise data loss will occur. - - Ensures given pool and RBD image exists, is mapped to a block device, - and the device is formatted and mounted at the given mount_point. - - If formatting a device for the first time, data existing at mount_point - will be migrated to the RBD device before being re-mounted. - - All services listed in system_services will be stopped prior to data - migration and restarted when complete. - """ - # Ensure pool, RBD image, RBD mappings are in place. - if not pool_exists(service, pool): - log('Creating new pool {}.'.format(pool), level=INFO) - create_pool(service, pool, replicas=replicas) - - if not rbd_exists(service, pool, rbd_img): - log('Creating RBD image ({}).'.format(rbd_img), level=INFO) - create_rbd_image(service, pool, rbd_img, sizemb) - - if not image_mapped(rbd_img): - log('Mapping RBD Image {} as a Block Device.'.format(rbd_img), - level=INFO) - map_block_storage(service, pool, rbd_img) - - # make file system - # TODO: What happens if for whatever reason this is run again and - # the data is already in the rbd device and/or is mounted?? - # When it is mounted already, it will fail to make the fs - # XXX: This is really sketchy! Need to at least add an fstab entry - # otherwise this hook will blow away existing data if its executed - # after a reboot. - if not filesystem_mounted(mount_point): - make_filesystem(blk_device, fstype) - - for svc in system_services: - if service_running(svc): - log('Stopping services {} prior to migrating data.' - .format(svc), level=DEBUG) - service_stop(svc) - - place_data_on_block_device(blk_device, mount_point) - - for svc in system_services: - log('Starting service {} after migrating data.' - .format(svc), level=DEBUG) - service_start(svc) - - -def ensure_ceph_keyring(service, user=None, group=None, - relation='ceph', key=None): - """Ensures a ceph keyring is created for a named service and optionally - ensures user and group ownership. - - @returns boolean: Flag to indicate whether a key was successfully written - to disk based on either relation data or a supplied key - """ - if not key: - for rid in relation_ids(relation): - for unit in related_units(rid): - key = relation_get('key', rid=rid, unit=unit) - if key: - break - - if not key: - return False - - create_keyring(service=service, key=key) - keyring = _keyring_path(service) - if user and group: - check_call(['chown', '%s.%s' % (user, group), keyring]) - - return True - - -def ceph_version(): - """Retrieve the local version of ceph.""" - if os.path.exists('/usr/bin/ceph'): - cmd = ['ceph', '-v'] - output = check_output(cmd) - if six.PY3: - output = output.decode('UTF-8') - output = output.split() - if len(output) > 3: - return output[2] - else: - return None - else: - return None - - -class CephBrokerRq(object): - """Ceph broker request. - - Multiple operations can be added to a request and sent to the Ceph broker - to be executed. - - Request is json-encoded for sending over the wire. - - The API is versioned and defaults to version 1. - """ - - def __init__(self, api_version=1, request_id=None): - self.api_version = api_version - if request_id: - self.request_id = request_id - else: - self.request_id = str(uuid.uuid1()) - self.ops = [] - - def add_op_request_access_to_group(self, name, namespace=None, - permission=None, key_name=None): - """ - Adds the requested permissions to the current service's Ceph key, - allowing the key to access only the specified pools - """ - self.ops.append({'op': 'add-permissions-to-key', 'group': name, - 'namespace': namespace, 'name': key_name or service_name(), - 'group-permission': permission}) - - def add_op_create_pool(self, name, replica_count=3, pg_num=None, - weight=None, group=None, namespace=None): - """Adds an operation to create a pool. - - @param pg_num setting: optional setting. If not provided, this value - will be calculated by the broker based on how many OSDs are in the - cluster at the time of creation. Note that, if provided, this value - will be capped at the current available maximum. - @param weight: the percentage of data the pool makes up - """ - if pg_num and weight: - raise ValueError('pg_num and weight are mutually exclusive') - - self.ops.append({'op': 'create-pool', 'name': name, - 'replicas': replica_count, 'pg_num': pg_num, - 'weight': weight, 'group': group, - 'group-namespace': namespace}) - - def set_ops(self, ops): - """Set request ops to provided value. - - Useful for injecting ops that come from a previous request - to allow comparisons to ensure validity. - """ - self.ops = ops - - @property - def request(self): - return json.dumps({'api-version': self.api_version, 'ops': self.ops, - 'request-id': self.request_id}) - - def _ops_equal(self, other): - if len(self.ops) == len(other.ops): - for req_no in range(0, len(self.ops)): - for key in ['replicas', 'name', 'op', 'pg_num', 'weight']: - if self.ops[req_no].get(key) != other.ops[req_no].get(key): - return False - else: - return False - return True - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - if self.api_version == other.api_version and \ - self._ops_equal(other): - return True - else: - return False - - def __ne__(self, other): - return not self.__eq__(other) - - -class CephBrokerRsp(object): - """Ceph broker response. - - Response is json-decoded and contents provided as methods/properties. - - The API is versioned and defaults to version 1. - """ - - def __init__(self, encoded_rsp): - self.api_version = None - self.rsp = json.loads(encoded_rsp) - - @property - def request_id(self): - return self.rsp.get('request-id') - - @property - def exit_code(self): - return self.rsp.get('exit-code') - - @property - def exit_msg(self): - return self.rsp.get('stderr') - - -# Ceph Broker Conversation: -# If a charm needs an action to be taken by ceph it can create a CephBrokerRq -# and send that request to ceph via the ceph relation. The CephBrokerRq has a -# unique id so that the client can identity which CephBrokerRsp is associated -# with the request. Ceph will also respond to each client unit individually -# creating a response key per client unit eg glance/0 will get a CephBrokerRsp -# via key broker-rsp-glance-0 -# -# To use this the charm can just do something like: -# -# from charmhelpers.contrib.storage.linux.ceph import ( -# send_request_if_needed, -# is_request_complete, -# CephBrokerRq, -# ) -# -# @hooks.hook('ceph-relation-changed') -# def ceph_changed(): -# rq = CephBrokerRq() -# rq.add_op_create_pool(name='poolname', replica_count=3) -# -# if is_request_complete(rq): -# -# else: -# send_request_if_needed(get_ceph_request()) -# -# CephBrokerRq and CephBrokerRsp are serialized into JSON. Below is an example -# of glance having sent a request to ceph which ceph has successfully processed -# 'ceph:8': { -# 'ceph/0': { -# 'auth': 'cephx', -# 'broker-rsp-glance-0': '{"request-id": "0bc7dc54", "exit-code": 0}', -# 'broker_rsp': '{"request-id": "0da543b8", "exit-code": 0}', -# 'ceph-public-address': '10.5.44.103', -# 'key': 'AQCLDttVuHXINhAAvI144CB09dYchhHyTUY9BQ==', -# 'private-address': '10.5.44.103', -# }, -# 'glance/0': { -# 'broker_req': ('{"api-version": 1, "request-id": "0bc7dc54", ' -# '"ops": [{"replicas": 3, "name": "glance", ' -# '"op": "create-pool"}]}'), -# 'private-address': '10.5.44.109', -# }, -# } - -def get_previous_request(rid): - """Return the last ceph broker request sent on a given relation - - @param rid: Relation id to query for request - """ - request = None - broker_req = relation_get(attribute='broker_req', rid=rid, - unit=local_unit()) - if broker_req: - request_data = json.loads(broker_req) - request = CephBrokerRq(api_version=request_data['api-version'], - request_id=request_data['request-id']) - request.set_ops(request_data['ops']) - - return request - - -def get_request_states(request, relation='ceph'): - """Return a dict of requests per relation id with their corresponding - completion state. - - This allows a charm, which has a request for ceph, to see whether there is - an equivalent request already being processed and if so what state that - request is in. - - @param request: A CephBrokerRq object - """ - complete = [] - requests = {} - for rid in relation_ids(relation): - complete = False - previous_request = get_previous_request(rid) - if request == previous_request: - sent = True - complete = is_request_complete_for_rid(previous_request, rid) - else: - sent = False - complete = False - - requests[rid] = { - 'sent': sent, - 'complete': complete, - } - - return requests - - -def is_request_sent(request, relation='ceph'): - """Check to see if a functionally equivalent request has already been sent - - Returns True if a similair request has been sent - - @param request: A CephBrokerRq object - """ - states = get_request_states(request, relation=relation) - for rid in states.keys(): - if not states[rid]['sent']: - return False - - return True - - -def is_request_complete(request, relation='ceph'): - """Check to see if a functionally equivalent request has already been - completed - - Returns True if a similair request has been completed - - @param request: A CephBrokerRq object - """ - states = get_request_states(request, relation=relation) - for rid in states.keys(): - if not states[rid]['complete']: - return False - - return True - - -def is_request_complete_for_rid(request, rid): - """Check if a given request has been completed on the given relation - - @param request: A CephBrokerRq object - @param rid: Relation ID - """ - broker_key = get_broker_rsp_key() - for unit in related_units(rid): - rdata = relation_get(rid=rid, unit=unit) - if rdata.get(broker_key): - rsp = CephBrokerRsp(rdata.get(broker_key)) - if rsp.request_id == request.request_id: - if not rsp.exit_code: - return True - else: - # The remote unit sent no reply targeted at this unit so either the - # remote ceph cluster does not support unit targeted replies or it - # has not processed our request yet. - if rdata.get('broker_rsp'): - request_data = json.loads(rdata['broker_rsp']) - if request_data.get('request-id'): - log('Ignoring legacy broker_rsp without unit key as remote ' - 'service supports unit specific replies', level=DEBUG) - else: - log('Using legacy broker_rsp as remote service does not ' - 'supports unit specific replies', level=DEBUG) - rsp = CephBrokerRsp(rdata['broker_rsp']) - if not rsp.exit_code: - return True - - return False - - -def get_broker_rsp_key(): - """Return broker response key for this unit - - This is the key that ceph is going to use to pass request status - information back to this unit - """ - return 'broker-rsp-' + local_unit().replace('/', '-') - - -def send_request_if_needed(request, relation='ceph'): - """Send broker request if an equivalent request has not already been sent - - @param request: A CephBrokerRq object - """ - if is_request_sent(request, relation=relation): - log('Request already sent but not complete, not sending new request', - level=DEBUG) - else: - for rid in relation_ids(relation): - log('Sending request {}'.format(request.request_id), level=DEBUG) - relation_set(relation_id=rid, broker_req=request.request) - - -def is_broker_action_done(action, rid=None, unit=None): - """Check whether broker action has completed yet. - - @param action: name of action to be performed - @returns True if action complete otherwise False - """ - rdata = relation_get(rid, unit) or {} - broker_rsp = rdata.get(get_broker_rsp_key()) - if not broker_rsp: - return False - - rsp = CephBrokerRsp(broker_rsp) - unit_name = local_unit().partition('/')[2] - key = "unit_{}_ceph_broker_action.{}".format(unit_name, action) - kvstore = kv() - val = kvstore.get(key=key) - if val and val == rsp.request_id: - return True - - return False - - -def mark_broker_action_done(action, rid=None, unit=None): - """Mark action as having been completed. - - @param action: name of action to be performed - @returns None - """ - rdata = relation_get(rid, unit) or {} - broker_rsp = rdata.get(get_broker_rsp_key()) - if not broker_rsp: - return - - rsp = CephBrokerRsp(broker_rsp) - unit_name = local_unit().partition('/')[2] - key = "unit_{}_ceph_broker_action.{}".format(unit_name, action) - kvstore = kv() - kvstore.set(key=key, value=rsp.request_id) - kvstore.flush() - - -class CephConfContext(object): - """Ceph config (ceph.conf) context. - - Supports user-provided Ceph configuration settings. Use can provide a - dictionary as the value for the config-flags charm option containing - Ceph configuration settings keyede by their section in ceph.conf. - """ - def __init__(self, permitted_sections=None): - self.permitted_sections = permitted_sections or [] - - def __call__(self): - conf = config('config-flags') - if not conf: - return {} - - conf = config_flags_parser(conf) - if not isinstance(conf, dict): - log("Provided config-flags is not a dictionary - ignoring", - level=WARNING) - return {} - - permitted = self.permitted_sections - if permitted: - diff = set(conf.keys()).difference(set(permitted)) - if diff: - log("Config-flags contains invalid keys '%s' - they will be " - "ignored" % (', '.join(diff)), level=WARNING) - - ceph_conf = {} - for key in conf: - if permitted and key not in permitted: - log("Ignoring key '%s'" % key, level=WARNING) - continue - - ceph_conf[key] = conf[key] - - return ceph_conf diff --git a/hooks/charmhelpers/contrib/storage/linux/loopback.py b/hooks/charmhelpers/contrib/storage/linux/loopback.py deleted file mode 100644 index 1d6ae6f..0000000 --- a/hooks/charmhelpers/contrib/storage/linux/loopback.py +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import re -from subprocess import ( - check_call, - check_output, -) - -import six - - -################################################## -# loopback device helpers. -################################################## -def loopback_devices(): - ''' - Parse through 'losetup -a' output to determine currently mapped - loopback devices. Output is expected to look like: - - /dev/loop0: [0807]:961814 (/tmp/my.img) - - :returns: dict: a dict mapping {loopback_dev: backing_file} - ''' - loopbacks = {} - cmd = ['losetup', '-a'] - devs = [d.strip().split(' ') for d in - check_output(cmd).splitlines() if d != ''] - for dev, _, f in devs: - loopbacks[dev.replace(':', '')] = re.search('\((\S+)\)', f).groups()[0] - return loopbacks - - -def create_loopback(file_path): - ''' - Create a loopback device for a given backing file. - - :returns: str: Full path to new loopback device (eg, /dev/loop0) - ''' - file_path = os.path.abspath(file_path) - check_call(['losetup', '--find', file_path]) - for d, f in six.iteritems(loopback_devices()): - if f == file_path: - return d - - -def ensure_loopback_device(path, size): - ''' - Ensure a loopback device exists for a given backing file path and size. - If it a loopback device is not mapped to file, a new one will be created. - - TODO: Confirm size of found loopback device. - - :returns: str: Full path to the ensured loopback device (eg, /dev/loop0) - ''' - for d, f in six.iteritems(loopback_devices()): - if f == path: - return d - - if not os.path.exists(path): - cmd = ['truncate', '--size', size, path] - check_call(cmd) - - return create_loopback(path) - - -def is_mapped_loopback_device(device): - """ - Checks if a given device name is an existing/mapped loopback device. - :param device: str: Full path to the device (eg, /dev/loop1). - :returns: str: Path to the backing file if is a loopback device - empty string otherwise - """ - return loopback_devices().get(device, "") diff --git a/hooks/charmhelpers/contrib/storage/linux/lvm.py b/hooks/charmhelpers/contrib/storage/linux/lvm.py deleted file mode 100644 index 79a7a24..0000000 --- a/hooks/charmhelpers/contrib/storage/linux/lvm.py +++ /dev/null @@ -1,153 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import functools -from subprocess import ( - CalledProcessError, - check_call, - check_output, - Popen, - PIPE, -) - - -################################################## -# LVM helpers. -################################################## -def deactivate_lvm_volume_group(block_device): - ''' - Deactivate any volume gruop associated with an LVM physical volume. - - :param block_device: str: Full path to LVM physical volume - ''' - vg = list_lvm_volume_group(block_device) - if vg: - cmd = ['vgchange', '-an', vg] - check_call(cmd) - - -def is_lvm_physical_volume(block_device): - ''' - Determine whether a block device is initialized as an LVM PV. - - :param block_device: str: Full path of block device to inspect. - - :returns: boolean: True if block device is a PV, False if not. - ''' - try: - check_output(['pvdisplay', block_device]) - return True - except CalledProcessError: - return False - - -def remove_lvm_physical_volume(block_device): - ''' - Remove LVM PV signatures from a given block device. - - :param block_device: str: Full path of block device to scrub. - ''' - p = Popen(['pvremove', '-ff', block_device], - stdin=PIPE) - p.communicate(input='y\n') - - -def list_lvm_volume_group(block_device): - ''' - List LVM volume group associated with a given block device. - - Assumes block device is a valid LVM PV. - - :param block_device: str: Full path of block device to inspect. - - :returns: str: Name of volume group associated with block device or None - ''' - vg = None - pvd = check_output(['pvdisplay', block_device]).splitlines() - for lvm in pvd: - lvm = lvm.decode('UTF-8') - if lvm.strip().startswith('VG Name'): - vg = ' '.join(lvm.strip().split()[2:]) - return vg - - -def create_lvm_physical_volume(block_device): - ''' - Initialize a block device as an LVM physical volume. - - :param block_device: str: Full path of block device to initialize. - - ''' - check_call(['pvcreate', block_device]) - - -def create_lvm_volume_group(volume_group, block_device): - ''' - Create an LVM volume group backed by a given block device. - - Assumes block device has already been initialized as an LVM PV. - - :param volume_group: str: Name of volume group to create. - :block_device: str: Full path of PV-initialized block device. - ''' - check_call(['vgcreate', volume_group, block_device]) - - -def list_logical_volumes(select_criteria=None, path_mode=False): - ''' - List logical volumes - - :param select_criteria: str: Limit list to those volumes matching this - criteria (see 'lvs -S help' for more details) - :param path_mode: bool: return logical volume name in 'vg/lv' format, this - format is required for some commands like lvextend - :returns: [str]: List of logical volumes - ''' - lv_diplay_attr = 'lv_name' - if path_mode: - # Parsing output logic relies on the column order - lv_diplay_attr = 'vg_name,' + lv_diplay_attr - cmd = ['lvs', '--options', lv_diplay_attr, '--noheadings'] - if select_criteria: - cmd.extend(['--select', select_criteria]) - lvs = [] - for lv in check_output(cmd).decode('UTF-8').splitlines(): - if not lv: - continue - if path_mode: - lvs.append('/'.join(lv.strip().split())) - else: - lvs.append(lv.strip()) - return lvs - - -list_thin_logical_volume_pools = functools.partial( - list_logical_volumes, - select_criteria='lv_attr =~ ^t') - -list_thin_logical_volumes = functools.partial( - list_logical_volumes, - select_criteria='lv_attr =~ ^V') - - -def extend_logical_volume_by_device(lv_name, block_device): - ''' - Extends the size of logical volume lv_name by the amount of free space on - physical volume block_device. - - :param lv_name: str: name of logical volume to be extended (vg/lv format) - :param block_device: str: name of block_device to be allocated to lv_name - ''' - cmd = ['lvextend', lv_name, block_device] - check_call(cmd) diff --git a/hooks/charmhelpers/contrib/storage/linux/utils.py b/hooks/charmhelpers/contrib/storage/linux/utils.py deleted file mode 100644 index c942889..0000000 --- a/hooks/charmhelpers/contrib/storage/linux/utils.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import re -from stat import S_ISBLK - -from subprocess import ( - check_call, - check_output, - call -) - - -def is_block_device(path): - ''' - Confirm device at path is a valid block device node. - - :returns: boolean: True if path is a block device, False if not. - ''' - if not os.path.exists(path): - return False - return S_ISBLK(os.stat(path).st_mode) - - -def zap_disk(block_device): - ''' - Clear a block device of partition table. Relies on sgdisk, which is - installed as pat of the 'gdisk' package in Ubuntu. - - :param block_device: str: Full path of block device to clean. - ''' - # https://github.com/ceph/ceph/commit/fdd7f8d83afa25c4e09aaedd90ab93f3b64a677b - # sometimes sgdisk exits non-zero; this is OK, dd will clean up - call(['sgdisk', '--zap-all', '--', block_device]) - call(['sgdisk', '--clear', '--mbrtogpt', '--', block_device]) - dev_end = check_output(['blockdev', '--getsz', - block_device]).decode('UTF-8') - gpt_end = int(dev_end.split()[0]) - 100 - check_call(['dd', 'if=/dev/zero', 'of=%s' % (block_device), - 'bs=1M', 'count=1']) - check_call(['dd', 'if=/dev/zero', 'of=%s' % (block_device), - 'bs=512', 'count=100', 'seek=%s' % (gpt_end)]) - - -def is_device_mounted(device): - '''Given a device path, return True if that device is mounted, and False - if it isn't. - - :param device: str: Full path of the device to check. - :returns: boolean: True if the path represents a mounted device, False if - it doesn't. - ''' - try: - out = check_output(['lsblk', '-P', device]).decode('UTF-8') - except Exception: - return False - return bool(re.search(r'MOUNTPOINT=".+"', out)) diff --git a/hooks/charmhelpers/core/__init__.py b/hooks/charmhelpers/core/__init__.py deleted file mode 100644 index d7567b8..0000000 --- a/hooks/charmhelpers/core/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/hooks/charmhelpers/core/decorators.py b/hooks/charmhelpers/core/decorators.py deleted file mode 100644 index 6ad41ee..0000000 --- a/hooks/charmhelpers/core/decorators.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# -# Copyright 2014 Canonical Ltd. -# -# Authors: -# Edward Hope-Morley -# - -import time - -from charmhelpers.core.hookenv import ( - log, - INFO, -) - - -def retry_on_exception(num_retries, base_delay=0, exc_type=Exception): - """If the decorated function raises exception exc_type, allow num_retries - retry attempts before raise the exception. - """ - def _retry_on_exception_inner_1(f): - def _retry_on_exception_inner_2(*args, **kwargs): - retries = num_retries - multiplier = 1 - while True: - try: - return f(*args, **kwargs) - except exc_type: - if not retries: - raise - - delay = base_delay * multiplier - multiplier += 1 - log("Retrying '%s' %d more times (delay=%s)" % - (f.__name__, retries, delay), level=INFO) - retries -= 1 - if delay: - time.sleep(delay) - - return _retry_on_exception_inner_2 - - return _retry_on_exception_inner_1 diff --git a/hooks/charmhelpers/core/files.py b/hooks/charmhelpers/core/files.py deleted file mode 100644 index fdd82b7..0000000 --- a/hooks/charmhelpers/core/files.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__author__ = 'Jorge Niedbalski ' - -import os -import subprocess - - -def sed(filename, before, after, flags='g'): - """ - Search and replaces the given pattern on filename. - - :param filename: relative or absolute file path. - :param before: expression to be replaced (see 'man sed') - :param after: expression to replace with (see 'man sed') - :param flags: sed-compatible regex flags in example, to make - the search and replace case insensitive, specify ``flags="i"``. - The ``g`` flag is always specified regardless, so you do not - need to remember to include it when overriding this parameter. - :returns: If the sed command exit code was zero then return, - otherwise raise CalledProcessError. - """ - expression = r's/{0}/{1}/{2}'.format(before, - after, flags) - - return subprocess.check_call(["sed", "-i", "-r", "-e", - expression, - os.path.expanduser(filename)]) diff --git a/hooks/charmhelpers/core/fstab.py b/hooks/charmhelpers/core/fstab.py deleted file mode 100644 index d9fa915..0000000 --- a/hooks/charmhelpers/core/fstab.py +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import io -import os - -__author__ = 'Jorge Niedbalski R. ' - - -class Fstab(io.FileIO): - """This class extends file in order to implement a file reader/writer - for file `/etc/fstab` - """ - - class Entry(object): - """Entry class represents a non-comment line on the `/etc/fstab` file - """ - def __init__(self, device, mountpoint, filesystem, - options, d=0, p=0): - self.device = device - self.mountpoint = mountpoint - self.filesystem = filesystem - - if not options: - options = "defaults" - - self.options = options - self.d = int(d) - self.p = int(p) - - def __eq__(self, o): - return str(self) == str(o) - - def __str__(self): - return "{} {} {} {} {} {}".format(self.device, - self.mountpoint, - self.filesystem, - self.options, - self.d, - self.p) - - DEFAULT_PATH = os.path.join(os.path.sep, 'etc', 'fstab') - - def __init__(self, path=None): - if path: - self._path = path - else: - self._path = self.DEFAULT_PATH - super(Fstab, self).__init__(self._path, 'rb+') - - def _hydrate_entry(self, line): - # NOTE: use split with no arguments to split on any - # whitespace including tabs - return Fstab.Entry(*filter( - lambda x: x not in ('', None), - line.strip("\n").split())) - - @property - def entries(self): - self.seek(0) - for line in self.readlines(): - line = line.decode('us-ascii') - try: - if line.strip() and not line.strip().startswith("#"): - yield self._hydrate_entry(line) - except ValueError: - pass - - def get_entry_by_attr(self, attr, value): - for entry in self.entries: - e_attr = getattr(entry, attr) - if e_attr == value: - return entry - return None - - def add_entry(self, entry): - if self.get_entry_by_attr('device', entry.device): - return False - - self.write((str(entry) + '\n').encode('us-ascii')) - self.truncate() - return entry - - def remove_entry(self, entry): - self.seek(0) - - lines = [l.decode('us-ascii') for l in self.readlines()] - - found = False - for index, line in enumerate(lines): - if line.strip() and not line.strip().startswith("#"): - if self._hydrate_entry(line) == entry: - found = True - break - - if not found: - return False - - lines.remove(line) - - self.seek(0) - self.write(''.join(lines).encode('us-ascii')) - self.truncate() - return True - - @classmethod - def remove_by_mountpoint(cls, mountpoint, path=None): - fstab = cls(path=path) - entry = fstab.get_entry_by_attr('mountpoint', mountpoint) - if entry: - return fstab.remove_entry(entry) - return False - - @classmethod - def add(cls, device, mountpoint, filesystem, options=None, path=None): - return cls(path=path).add_entry(Fstab.Entry(device, - mountpoint, filesystem, - options=options)) diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py deleted file mode 100644 index 5a88f79..0000000 --- a/hooks/charmhelpers/core/hookenv.py +++ /dev/null @@ -1,1206 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"Interactions with the Juju environment" -# Copyright 2013 Canonical Ltd. -# -# Authors: -# Charm Helpers Developers - -from __future__ import print_function -import copy -from distutils.version import LooseVersion -from functools import wraps -from collections import namedtuple -import glob -import os -import json -import yaml -import subprocess -import sys -import errno -import tempfile -from subprocess import CalledProcessError - -import six -if not six.PY3: - from UserDict import UserDict -else: - from collections import UserDict - -CRITICAL = "CRITICAL" -ERROR = "ERROR" -WARNING = "WARNING" -INFO = "INFO" -DEBUG = "DEBUG" -TRACE = "TRACE" -MARKER = object() - -cache = {} - - -def cached(func): - """Cache return values for multiple executions of func + args - - For example:: - - @cached - def unit_get(attribute): - pass - - unit_get('test') - - will cache the result of unit_get + 'test' for future calls. - """ - @wraps(func) - def wrapper(*args, **kwargs): - global cache - key = str((func, args, kwargs)) - try: - return cache[key] - except KeyError: - pass # Drop out of the exception handler scope. - res = func(*args, **kwargs) - cache[key] = res - return res - wrapper._wrapped = func - return wrapper - - -def flush(key): - """Flushes any entries from function cache where the - key is found in the function+args """ - flush_list = [] - for item in cache: - if key in item: - flush_list.append(item) - for item in flush_list: - del cache[item] - - -def log(message, level=None): - """Write a message to the juju log""" - command = ['juju-log'] - if level: - command += ['-l', level] - if not isinstance(message, six.string_types): - message = repr(message) - command += [message] - # Missing juju-log should not cause failures in unit tests - # Send log output to stderr - try: - subprocess.call(command) - except OSError as e: - if e.errno == errno.ENOENT: - if level: - message = "{}: {}".format(level, message) - message = "juju-log: {}".format(message) - print(message, file=sys.stderr) - else: - raise - - -class Serializable(UserDict): - """Wrapper, an object that can be serialized to yaml or json""" - - def __init__(self, obj): - # wrap the object - UserDict.__init__(self) - self.data = obj - - def __getattr__(self, attr): - # See if this object has attribute. - if attr in ("json", "yaml", "data"): - return self.__dict__[attr] - # Check for attribute in wrapped object. - got = getattr(self.data, attr, MARKER) - if got is not MARKER: - return got - # Proxy to the wrapped object via dict interface. - try: - return self.data[attr] - except KeyError: - raise AttributeError(attr) - - def __getstate__(self): - # Pickle as a standard dictionary. - return self.data - - def __setstate__(self, state): - # Unpickle into our wrapper. - self.data = state - - def json(self): - """Serialize the object to json""" - return json.dumps(self.data) - - def yaml(self): - """Serialize the object to yaml""" - return yaml.dump(self.data) - - -def execution_environment(): - """A convenient bundling of the current execution context""" - context = {} - context['conf'] = config() - if relation_id(): - context['reltype'] = relation_type() - context['relid'] = relation_id() - context['rel'] = relation_get() - context['unit'] = local_unit() - context['rels'] = relations() - context['env'] = os.environ - return context - - -def in_relation_hook(): - """Determine whether we're running in a relation hook""" - return 'JUJU_RELATION' in os.environ - - -def relation_type(): - """The scope for the current relation hook""" - return os.environ.get('JUJU_RELATION', None) - - -@cached -def relation_id(relation_name=None, service_or_unit=None): - """The relation ID for the current or a specified relation""" - if not relation_name and not service_or_unit: - return os.environ.get('JUJU_RELATION_ID', None) - elif relation_name and service_or_unit: - service_name = service_or_unit.split('/')[0] - for relid in relation_ids(relation_name): - remote_service = remote_service_name(relid) - if remote_service == service_name: - return relid - else: - raise ValueError('Must specify neither or both of relation_name and service_or_unit') - - -def local_unit(): - """Local unit ID""" - return os.environ['JUJU_UNIT_NAME'] - - -def remote_unit(): - """The remote unit for the current relation hook""" - return os.environ.get('JUJU_REMOTE_UNIT', None) - - -def service_name(): - """The name service group this unit belongs to""" - return local_unit().split('/')[0] - - -def principal_unit(): - """Returns the principal unit of this unit, otherwise None""" - # Juju 2.2 and above provides JUJU_PRINCIPAL_UNIT - principal_unit = os.environ.get('JUJU_PRINCIPAL_UNIT', None) - # If it's empty, then this unit is the principal - if principal_unit == '': - return os.environ['JUJU_UNIT_NAME'] - elif principal_unit is not None: - return principal_unit - # For Juju 2.1 and below, let's try work out the principle unit by - # the various charms' metadata.yaml. - for reltype in relation_types(): - for rid in relation_ids(reltype): - for unit in related_units(rid): - md = _metadata_unit(unit) - if not md: - continue - subordinate = md.pop('subordinate', None) - if not subordinate: - return unit - return None - - -@cached -def remote_service_name(relid=None): - """The remote service name for a given relation-id (or the current relation)""" - if relid is None: - unit = remote_unit() - else: - units = related_units(relid) - unit = units[0] if units else None - return unit.split('/')[0] if unit else None - - -def hook_name(): - """The name of the currently executing hook""" - return os.environ.get('JUJU_HOOK_NAME', os.path.basename(sys.argv[0])) - - -class Config(dict): - """A dictionary representation of the charm's config.yaml, with some - extra features: - - - See which values in the dictionary have changed since the previous hook. - - For values that have changed, see what the previous value was. - - Store arbitrary data for use in a later hook. - - NOTE: Do not instantiate this object directly - instead call - ``hookenv.config()``, which will return an instance of :class:`Config`. - - Example usage:: - - >>> # inside a hook - >>> from charmhelpers.core import hookenv - >>> config = hookenv.config() - >>> config['foo'] - 'bar' - >>> # store a new key/value for later use - >>> config['mykey'] = 'myval' - - - >>> # user runs `juju set mycharm foo=baz` - >>> # now we're inside subsequent config-changed hook - >>> config = hookenv.config() - >>> config['foo'] - 'baz' - >>> # test to see if this val has changed since last hook - >>> config.changed('foo') - True - >>> # what was the previous value? - >>> config.previous('foo') - 'bar' - >>> # keys/values that we add are preserved across hooks - >>> config['mykey'] - 'myval' - - """ - CONFIG_FILE_NAME = '.juju-persistent-config' - - def __init__(self, *args, **kw): - super(Config, self).__init__(*args, **kw) - self.implicit_save = True - self._prev_dict = None - self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME) - if os.path.exists(self.path): - self.load_previous() - atexit(self._implicit_save) - - def load_previous(self, path=None): - """Load previous copy of config from disk. - - In normal usage you don't need to call this method directly - it - is called automatically at object initialization. - - :param path: - - File path from which to load the previous config. If `None`, - config is loaded from the default location. If `path` is - specified, subsequent `save()` calls will write to the same - path. - - """ - self.path = path or self.path - with open(self.path) as f: - self._prev_dict = json.load(f) - for k, v in copy.deepcopy(self._prev_dict).items(): - if k not in self: - self[k] = v - - def changed(self, key): - """Return True if the current value for this key is different from - the previous value. - - """ - if self._prev_dict is None: - return True - return self.previous(key) != self.get(key) - - def previous(self, key): - """Return previous value for this key, or None if there - is no previous value. - - """ - if self._prev_dict: - return self._prev_dict.get(key) - return None - - def save(self): - """Save this config to disk. - - If the charm is using the :mod:`Services Framework ` - or :meth:'@hook ' decorator, this - is called automatically at the end of successful hook execution. - Otherwise, it should be called directly by user code. - - To disable automatic saves, set ``implicit_save=False`` on this - instance. - - """ - with open(self.path, 'w') as f: - json.dump(self, f) - - def _implicit_save(self): - if self.implicit_save: - self.save() - - -@cached -def config(scope=None): - """Juju charm configuration""" - config_cmd_line = ['config-get'] - if scope is not None: - config_cmd_line.append(scope) - else: - config_cmd_line.append('--all') - config_cmd_line.append('--format=json') - try: - config_data = json.loads( - subprocess.check_output(config_cmd_line).decode('UTF-8')) - if scope is not None: - return config_data - return Config(config_data) - except ValueError: - return None - - -@cached -def relation_get(attribute=None, unit=None, rid=None): - """Get relation information""" - _args = ['relation-get', '--format=json'] - if rid: - _args.append('-r') - _args.append(rid) - _args.append(attribute or '-') - if unit: - _args.append(unit) - try: - return json.loads(subprocess.check_output(_args).decode('UTF-8')) - except ValueError: - return None - except CalledProcessError as e: - if e.returncode == 2: - return None - raise - - -def relation_set(relation_id=None, relation_settings=None, **kwargs): - """Set relation information for the current unit""" - relation_settings = relation_settings if relation_settings else {} - relation_cmd_line = ['relation-set'] - accepts_file = "--file" in subprocess.check_output( - relation_cmd_line + ["--help"], universal_newlines=True) - if relation_id is not None: - relation_cmd_line.extend(('-r', relation_id)) - settings = relation_settings.copy() - settings.update(kwargs) - for key, value in settings.items(): - # Force value to be a string: it always should, but some call - # sites pass in things like dicts or numbers. - if value is not None: - settings[key] = "{}".format(value) - if accepts_file: - # --file was introduced in Juju 1.23.2. Use it by default if - # available, since otherwise we'll break if the relation data is - # too big. Ideally we should tell relation-set to read the data from - # stdin, but that feature is broken in 1.23.2: Bug #1454678. - with tempfile.NamedTemporaryFile(delete=False) as settings_file: - settings_file.write(yaml.safe_dump(settings).encode("utf-8")) - subprocess.check_call( - relation_cmd_line + ["--file", settings_file.name]) - os.remove(settings_file.name) - else: - for key, value in settings.items(): - if value is None: - relation_cmd_line.append('{}='.format(key)) - else: - relation_cmd_line.append('{}={}'.format(key, value)) - subprocess.check_call(relation_cmd_line) - # Flush cache of any relation-gets for local unit - flush(local_unit()) - - -def relation_clear(r_id=None): - ''' Clears any relation data already set on relation r_id ''' - settings = relation_get(rid=r_id, - unit=local_unit()) - for setting in settings: - if setting not in ['public-address', 'private-address']: - settings[setting] = None - relation_set(relation_id=r_id, - **settings) - - -@cached -def relation_ids(reltype=None): - """A list of relation_ids""" - reltype = reltype or relation_type() - relid_cmd_line = ['relation-ids', '--format=json'] - if reltype is not None: - relid_cmd_line.append(reltype) - return json.loads( - subprocess.check_output(relid_cmd_line).decode('UTF-8')) or [] - return [] - - -@cached -def related_units(relid=None): - """A list of related units""" - relid = relid or relation_id() - units_cmd_line = ['relation-list', '--format=json'] - if relid is not None: - units_cmd_line.extend(('-r', relid)) - return json.loads( - subprocess.check_output(units_cmd_line).decode('UTF-8')) or [] - - -@cached -def relation_for_unit(unit=None, rid=None): - """Get the json represenation of a unit's relation""" - unit = unit or remote_unit() - relation = relation_get(unit=unit, rid=rid) - for key in relation: - if key.endswith('-list'): - relation[key] = relation[key].split() - relation['__unit__'] = unit - return relation - - -@cached -def relations_for_id(relid=None): - """Get relations of a specific relation ID""" - relation_data = [] - relid = relid or relation_ids() - for unit in related_units(relid): - unit_data = relation_for_unit(unit, relid) - unit_data['__relid__'] = relid - relation_data.append(unit_data) - return relation_data - - -@cached -def relations_of_type(reltype=None): - """Get relations of a specific type""" - relation_data = [] - reltype = reltype or relation_type() - for relid in relation_ids(reltype): - for relation in relations_for_id(relid): - relation['__relid__'] = relid - relation_data.append(relation) - return relation_data - - -@cached -def metadata(): - """Get the current charm metadata.yaml contents as a python object""" - with open(os.path.join(charm_dir(), 'metadata.yaml')) as md: - return yaml.safe_load(md) - - -def _metadata_unit(unit): - """Given the name of a unit (e.g. apache2/0), get the unit charm's - metadata.yaml. Very similar to metadata() but allows us to inspect - other units. Unit needs to be co-located, such as a subordinate or - principal/primary. - - :returns: metadata.yaml as a python object. - - """ - basedir = os.sep.join(charm_dir().split(os.sep)[:-2]) - unitdir = 'unit-{}'.format(unit.replace(os.sep, '-')) - joineddir = os.path.join(basedir, unitdir, 'charm', 'metadata.yaml') - if not os.path.exists(joineddir): - return None - with open(joineddir) as md: - return yaml.safe_load(md) - - -@cached -def relation_types(): - """Get a list of relation types supported by this charm""" - rel_types = [] - md = metadata() - for key in ('provides', 'requires', 'peers'): - section = md.get(key) - if section: - rel_types.extend(section.keys()) - return rel_types - - -@cached -def peer_relation_id(): - '''Get the peers relation id if a peers relation has been joined, else None.''' - md = metadata() - section = md.get('peers') - if section: - for key in section: - relids = relation_ids(key) - if relids: - return relids[0] - return None - - -@cached -def relation_to_interface(relation_name): - """ - Given the name of a relation, return the interface that relation uses. - - :returns: The interface name, or ``None``. - """ - return relation_to_role_and_interface(relation_name)[1] - - -@cached -def relation_to_role_and_interface(relation_name): - """ - Given the name of a relation, return the role and the name of the interface - that relation uses (where role is one of ``provides``, ``requires``, or ``peers``). - - :returns: A tuple containing ``(role, interface)``, or ``(None, None)``. - """ - _metadata = metadata() - for role in ('provides', 'requires', 'peers'): - interface = _metadata.get(role, {}).get(relation_name, {}).get('interface') - if interface: - return role, interface - return None, None - - -@cached -def role_and_interface_to_relations(role, interface_name): - """ - Given a role and interface name, return a list of relation names for the - current charm that use that interface under that role (where role is one - of ``provides``, ``requires``, or ``peers``). - - :returns: A list of relation names. - """ - _metadata = metadata() - results = [] - for relation_name, relation in _metadata.get(role, {}).items(): - if relation['interface'] == interface_name: - results.append(relation_name) - return results - - -@cached -def interface_to_relations(interface_name): - """ - Given an interface, return a list of relation names for the current - charm that use that interface. - - :returns: A list of relation names. - """ - results = [] - for role in ('provides', 'requires', 'peers'): - results.extend(role_and_interface_to_relations(role, interface_name)) - return results - - -@cached -def charm_name(): - """Get the name of the current charm as is specified on metadata.yaml""" - return metadata().get('name') - - -@cached -def relations(): - """Get a nested dictionary of relation data for all related units""" - rels = {} - for reltype in relation_types(): - relids = {} - for relid in relation_ids(reltype): - units = {local_unit(): relation_get(unit=local_unit(), rid=relid)} - for unit in related_units(relid): - reldata = relation_get(unit=unit, rid=relid) - units[unit] = reldata - relids[relid] = units - rels[reltype] = relids - return rels - - -@cached -def is_relation_made(relation, keys='private-address'): - ''' - Determine whether a relation is established by checking for - presence of key(s). If a list of keys is provided, they - must all be present for the relation to be identified as made - ''' - if isinstance(keys, str): - keys = [keys] - for r_id in relation_ids(relation): - for unit in related_units(r_id): - context = {} - for k in keys: - context[k] = relation_get(k, rid=r_id, - unit=unit) - if None not in context.values(): - return True - return False - - -def _port_op(op_name, port, protocol="TCP"): - """Open or close a service network port""" - _args = [op_name] - icmp = protocol.upper() == "ICMP" - if icmp: - _args.append(protocol) - else: - _args.append('{}/{}'.format(port, protocol)) - try: - subprocess.check_call(_args) - except subprocess.CalledProcessError: - # Older Juju pre 2.3 doesn't support ICMP - # so treat it as a no-op if it fails. - if not icmp: - raise - - -def open_port(port, protocol="TCP"): - """Open a service network port""" - _port_op('open-port', port, protocol) - - -def close_port(port, protocol="TCP"): - """Close a service network port""" - _port_op('close-port', port, protocol) - - -def open_ports(start, end, protocol="TCP"): - """Opens a range of service network ports""" - _args = ['open-port'] - _args.append('{}-{}/{}'.format(start, end, protocol)) - subprocess.check_call(_args) - - -def close_ports(start, end, protocol="TCP"): - """Close a range of service network ports""" - _args = ['close-port'] - _args.append('{}-{}/{}'.format(start, end, protocol)) - subprocess.check_call(_args) - - -def opened_ports(): - """Get the opened ports - - *Note that this will only show ports opened in a previous hook* - - :returns: Opened ports as a list of strings: ``['8080/tcp', '8081-8083/tcp']`` - """ - _args = ['opened-ports', '--format=json'] - return json.loads(subprocess.check_output(_args).decode('UTF-8')) - - -@cached -def unit_get(attribute): - """Get the unit ID for the remote unit""" - _args = ['unit-get', '--format=json', attribute] - try: - return json.loads(subprocess.check_output(_args).decode('UTF-8')) - except ValueError: - return None - - -def unit_public_ip(): - """Get this unit's public IP address""" - return unit_get('public-address') - - -def unit_private_ip(): - """Get this unit's private IP address""" - return unit_get('private-address') - - -@cached -def storage_get(attribute=None, storage_id=None): - """Get storage attributes""" - _args = ['storage-get', '--format=json'] - if storage_id: - _args.extend(('-s', storage_id)) - if attribute: - _args.append(attribute) - try: - return json.loads(subprocess.check_output(_args).decode('UTF-8')) - except ValueError: - return None - - -@cached -def storage_list(storage_name=None): - """List the storage IDs for the unit""" - _args = ['storage-list', '--format=json'] - if storage_name: - _args.append(storage_name) - try: - return json.loads(subprocess.check_output(_args).decode('UTF-8')) - except ValueError: - return None - except OSError as e: - import errno - if e.errno == errno.ENOENT: - # storage-list does not exist - return [] - raise - - -class UnregisteredHookError(Exception): - """Raised when an undefined hook is called""" - pass - - -class Hooks(object): - """A convenient handler for hook functions. - - Example:: - - hooks = Hooks() - - # register a hook, taking its name from the function name - @hooks.hook() - def install(): - pass # your code here - - # register a hook, providing a custom hook name - @hooks.hook("config-changed") - def config_changed(): - pass # your code here - - if __name__ == "__main__": - # execute a hook based on the name the program is called by - hooks.execute(sys.argv) - """ - - def __init__(self, config_save=None): - super(Hooks, self).__init__() - self._hooks = {} - - # For unknown reasons, we allow the Hooks constructor to override - # config().implicit_save. - if config_save is not None: - config().implicit_save = config_save - - def register(self, name, function): - """Register a hook""" - self._hooks[name] = function - - def execute(self, args): - """Execute a registered hook based on args[0]""" - _run_atstart() - hook_name = os.path.basename(args[0]) - if hook_name in self._hooks: - try: - self._hooks[hook_name]() - except SystemExit as x: - if x.code is None or x.code == 0: - _run_atexit() - raise - _run_atexit() - else: - raise UnregisteredHookError(hook_name) - - def hook(self, *hook_names): - """Decorator, registering them as hooks""" - def wrapper(decorated): - for hook_name in hook_names: - self.register(hook_name, decorated) - else: - self.register(decorated.__name__, decorated) - if '_' in decorated.__name__: - self.register( - decorated.__name__.replace('_', '-'), decorated) - return decorated - return wrapper - - -def charm_dir(): - """Return the root directory of the current charm""" - d = os.environ.get('JUJU_CHARM_DIR') - if d is not None: - return d - return os.environ.get('CHARM_DIR') - - -@cached -def action_get(key=None): - """Gets the value of an action parameter, or all key/value param pairs""" - cmd = ['action-get'] - if key is not None: - cmd.append(key) - cmd.append('--format=json') - action_data = json.loads(subprocess.check_output(cmd).decode('UTF-8')) - return action_data - - -def action_set(values): - """Sets the values to be returned after the action finishes""" - cmd = ['action-set'] - for k, v in list(values.items()): - cmd.append('{}={}'.format(k, v)) - subprocess.check_call(cmd) - - -def action_fail(message): - """Sets the action status to failed and sets the error message. - - The results set by action_set are preserved.""" - subprocess.check_call(['action-fail', message]) - - -def action_name(): - """Get the name of the currently executing action.""" - return os.environ.get('JUJU_ACTION_NAME') - - -def action_uuid(): - """Get the UUID of the currently executing action.""" - return os.environ.get('JUJU_ACTION_UUID') - - -def action_tag(): - """Get the tag for the currently executing action.""" - return os.environ.get('JUJU_ACTION_TAG') - - -def status_set(workload_state, message): - """Set the workload state with a message - - Use status-set to set the workload state with a message which is visible - to the user via juju status. If the status-set command is not found then - assume this is juju < 1.23 and juju-log the message unstead. - - workload_state -- valid juju workload state. - message -- status update message - """ - valid_states = ['maintenance', 'blocked', 'waiting', 'active'] - if workload_state not in valid_states: - raise ValueError( - '{!r} is not a valid workload state'.format(workload_state) - ) - cmd = ['status-set', workload_state, message] - try: - ret = subprocess.call(cmd) - if ret == 0: - return - except OSError as e: - if e.errno != errno.ENOENT: - raise - log_message = 'status-set failed: {} {}'.format(workload_state, - message) - log(log_message, level='INFO') - - -def status_get(): - """Retrieve the previously set juju workload state and message - - If the status-get command is not found then assume this is juju < 1.23 and - return 'unknown', "" - - """ - cmd = ['status-get', "--format=json", "--include-data"] - try: - raw_status = subprocess.check_output(cmd) - except OSError as e: - if e.errno == errno.ENOENT: - return ('unknown', "") - else: - raise - else: - status = json.loads(raw_status.decode("UTF-8")) - return (status["status"], status["message"]) - - -def translate_exc(from_exc, to_exc): - def inner_translate_exc1(f): - @wraps(f) - def inner_translate_exc2(*args, **kwargs): - try: - return f(*args, **kwargs) - except from_exc: - raise to_exc - - return inner_translate_exc2 - - return inner_translate_exc1 - - -def application_version_set(version): - """Charm authors may trigger this command from any hook to output what - version of the application is running. This could be a package version, - for instance postgres version 9.5. It could also be a build number or - version control revision identifier, for instance git sha 6fb7ba68. """ - - cmd = ['application-version-set'] - cmd.append(version) - try: - subprocess.check_call(cmd) - except OSError: - log("Application Version: {}".format(version)) - - -@translate_exc(from_exc=OSError, to_exc=NotImplementedError) -def is_leader(): - """Does the current unit hold the juju leadership - - Uses juju to determine whether the current unit is the leader of its peers - """ - cmd = ['is-leader', '--format=json'] - return json.loads(subprocess.check_output(cmd).decode('UTF-8')) - - -@translate_exc(from_exc=OSError, to_exc=NotImplementedError) -def leader_get(attribute=None): - """Juju leader get value(s)""" - cmd = ['leader-get', '--format=json'] + [attribute or '-'] - return json.loads(subprocess.check_output(cmd).decode('UTF-8')) - - -@translate_exc(from_exc=OSError, to_exc=NotImplementedError) -def leader_set(settings=None, **kwargs): - """Juju leader set value(s)""" - # Don't log secrets. - # log("Juju leader-set '%s'" % (settings), level=DEBUG) - cmd = ['leader-set'] - settings = settings or {} - settings.update(kwargs) - for k, v in settings.items(): - if v is None: - cmd.append('{}='.format(k)) - else: - cmd.append('{}={}'.format(k, v)) - subprocess.check_call(cmd) - - -@translate_exc(from_exc=OSError, to_exc=NotImplementedError) -def payload_register(ptype, klass, pid): - """ is used while a hook is running to let Juju know that a - payload has been started.""" - cmd = ['payload-register'] - for x in [ptype, klass, pid]: - cmd.append(x) - subprocess.check_call(cmd) - - -@translate_exc(from_exc=OSError, to_exc=NotImplementedError) -def payload_unregister(klass, pid): - """ is used while a hook is running to let Juju know - that a payload has been manually stopped. The and provided - must match a payload that has been previously registered with juju using - payload-register.""" - cmd = ['payload-unregister'] - for x in [klass, pid]: - cmd.append(x) - subprocess.check_call(cmd) - - -@translate_exc(from_exc=OSError, to_exc=NotImplementedError) -def payload_status_set(klass, pid, status): - """is used to update the current status of a registered payload. - The and provided must match a payload that has been previously - registered with juju using payload-register. The must be one of the - follow: starting, started, stopping, stopped""" - cmd = ['payload-status-set'] - for x in [klass, pid, status]: - cmd.append(x) - subprocess.check_call(cmd) - - -@translate_exc(from_exc=OSError, to_exc=NotImplementedError) -def resource_get(name): - """used to fetch the resource path of the given name. - - must match a name of defined resource in metadata.yaml - - returns either a path or False if resource not available - """ - if not name: - return False - - cmd = ['resource-get', name] - try: - return subprocess.check_output(cmd).decode('UTF-8') - except subprocess.CalledProcessError: - return False - - -@cached -def juju_version(): - """Full version string (eg. '1.23.3.1-trusty-amd64')""" - # Per https://bugs.launchpad.net/juju-core/+bug/1455368/comments/1 - jujud = glob.glob('/var/lib/juju/tools/machine-*/jujud')[0] - return subprocess.check_output([jujud, 'version'], - universal_newlines=True).strip() - - -@cached -def has_juju_version(minimum_version): - """Return True if the Juju version is at least the provided version""" - return LooseVersion(juju_version()) >= LooseVersion(minimum_version) - - -_atexit = [] -_atstart = [] - - -def atstart(callback, *args, **kwargs): - '''Schedule a callback to run before the main hook. - - Callbacks are run in the order they were added. - - This is useful for modules and classes to perform initialization - and inject behavior. In particular: - - - Run common code before all of your hooks, such as logging - the hook name or interesting relation data. - - Defer object or module initialization that requires a hook - context until we know there actually is a hook context, - making testing easier. - - Rather than requiring charm authors to include boilerplate to - invoke your helper's behavior, have it run automatically if - your object is instantiated or module imported. - - This is not at all useful after your hook framework as been launched. - ''' - global _atstart - _atstart.append((callback, args, kwargs)) - - -def atexit(callback, *args, **kwargs): - '''Schedule a callback to run on successful hook completion. - - Callbacks are run in the reverse order that they were added.''' - _atexit.append((callback, args, kwargs)) - - -def _run_atstart(): - '''Hook frameworks must invoke this before running the main hook body.''' - global _atstart - for callback, args, kwargs in _atstart: - callback(*args, **kwargs) - del _atstart[:] - - -def _run_atexit(): - '''Hook frameworks must invoke this after the main hook body has - successfully completed. Do not invoke it if the hook fails.''' - global _atexit - for callback, args, kwargs in reversed(_atexit): - callback(*args, **kwargs) - del _atexit[:] - - -@translate_exc(from_exc=OSError, to_exc=NotImplementedError) -def network_get_primary_address(binding): - ''' - Retrieve the primary network address for a named binding - - :param binding: string. The name of a relation of extra-binding - :return: string. The primary IP address for the named binding - :raise: NotImplementedError if run on Juju < 2.0 - ''' - cmd = ['network-get', '--primary-address', binding] - return subprocess.check_output(cmd).decode('UTF-8').strip() - - -@translate_exc(from_exc=OSError, to_exc=NotImplementedError) -def network_get(endpoint, relation_id=None): - """ - Retrieve the network details for a relation endpoint - - :param endpoint: string. The name of a relation endpoint - :param relation_id: int. The ID of the relation for the current context. - :return: dict. The loaded YAML output of the network-get query. - :raise: NotImplementedError if run on Juju < 2.1 - """ - cmd = ['network-get', endpoint, '--format', 'yaml'] - if relation_id: - cmd.append('-r') - cmd.append(relation_id) - try: - response = subprocess.check_output( - cmd, - stderr=subprocess.STDOUT).decode('UTF-8').strip() - except CalledProcessError as e: - # Early versions of Juju 2.0.x required the --primary-address argument. - # We catch that condition here and raise NotImplementedError since - # the requested semantics are not available - the caller can then - # use the network_get_primary_address() method instead. - if '--primary-address is currently required' in e.output.decode('UTF-8'): - raise NotImplementedError - raise - return yaml.safe_load(response) - - -def add_metric(*args, **kwargs): - """Add metric values. Values may be expressed with keyword arguments. For - metric names containing dashes, these may be expressed as one or more - 'key=value' positional arguments. May only be called from the collect-metrics - hook.""" - _args = ['add-metric'] - _kvpairs = [] - _kvpairs.extend(args) - _kvpairs.extend(['{}={}'.format(k, v) for k, v in kwargs.items()]) - _args.extend(sorted(_kvpairs)) - try: - subprocess.check_call(_args) - return - except EnvironmentError as e: - if e.errno != errno.ENOENT: - raise - log_message = 'add-metric failed: {}'.format(' '.join(_kvpairs)) - log(log_message, level='INFO') - - -def meter_status(): - """Get the meter status, if running in the meter-status-changed hook.""" - return os.environ.get('JUJU_METER_STATUS') - - -def meter_info(): - """Get the meter status information, if running in the meter-status-changed - hook.""" - return os.environ.get('JUJU_METER_INFO') - - -def iter_units_for_relation_name(relation_name): - """Iterate through all units in a relation - - Generator that iterates through all the units in a relation and yields - a named tuple with rid and unit field names. - - Usage: - data = [(u.rid, u.unit) - for u in iter_units_for_relation_name(relation_name)] - - :param relation_name: string relation name - :yield: Named Tuple with rid and unit field names - """ - RelatedUnit = namedtuple('RelatedUnit', 'rid, unit') - for rid in relation_ids(relation_name): - for unit in related_units(rid): - yield RelatedUnit(rid, unit) - - -def ingress_address(rid=None, unit=None): - """ - Retrieve the ingress-address from a relation when available. Otherwise, - return the private-address. This function is to be used on the consuming - side of the relation. - - Usage: - addresses = [ingress_address(rid=u.rid, unit=u.unit) - for u in iter_units_for_relation_name(relation_name)] - - :param rid: string relation id - :param unit: string unit name - :side effect: calls relation_get - :return: string IP address - """ - settings = relation_get(rid=rid, unit=unit) - return (settings.get('ingress-address') or - settings.get('private-address')) diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py deleted file mode 100644 index fd14d60..0000000 --- a/hooks/charmhelpers/core/host.py +++ /dev/null @@ -1,1021 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Tools for working with the host system""" -# Copyright 2012 Canonical Ltd. -# -# Authors: -# Nick Moffitt -# Matthew Wedgwood - -import os -import re -import pwd -import glob -import grp -import random -import string -import subprocess -import hashlib -import functools -import itertools -import six - -from contextlib import contextmanager -from collections import OrderedDict -from .hookenv import log, DEBUG, local_unit -from .fstab import Fstab -from charmhelpers.osplatform import get_platform - -__platform__ = get_platform() -if __platform__ == "ubuntu": - from charmhelpers.core.host_factory.ubuntu import ( - service_available, - add_new_group, - lsb_release, - cmp_pkgrevno, - CompareHostReleases, - ) # flake8: noqa -- ignore F401 for this import -elif __platform__ == "centos": - from charmhelpers.core.host_factory.centos import ( - service_available, - add_new_group, - lsb_release, - cmp_pkgrevno, - CompareHostReleases, - ) # flake8: noqa -- ignore F401 for this import - -UPDATEDB_PATH = '/etc/updatedb.conf' - -def service_start(service_name, **kwargs): - """Start a system service. - - The specified service name is managed via the system level init system. - Some init systems (e.g. upstart) require that additional arguments be - provided in order to directly control service instances whereas other init - systems allow for addressing instances of a service directly by name (e.g. - systemd). - - The kwargs allow for the additional parameters to be passed to underlying - init systems for those systems which require/allow for them. For example, - the ceph-osd upstart script requires the id parameter to be passed along - in order to identify which running daemon should be reloaded. The follow- - ing example stops the ceph-osd service for instance id=4: - - service_stop('ceph-osd', id=4) - - :param service_name: the name of the service to stop - :param **kwargs: additional parameters to pass to the init system when - managing services. These will be passed as key=value - parameters to the init system's commandline. kwargs - are ignored for systemd enabled systems. - """ - return service('start', service_name, **kwargs) - - -def service_stop(service_name, **kwargs): - """Stop a system service. - - The specified service name is managed via the system level init system. - Some init systems (e.g. upstart) require that additional arguments be - provided in order to directly control service instances whereas other init - systems allow for addressing instances of a service directly by name (e.g. - systemd). - - The kwargs allow for the additional parameters to be passed to underlying - init systems for those systems which require/allow for them. For example, - the ceph-osd upstart script requires the id parameter to be passed along - in order to identify which running daemon should be reloaded. The follow- - ing example stops the ceph-osd service for instance id=4: - - service_stop('ceph-osd', id=4) - - :param service_name: the name of the service to stop - :param **kwargs: additional parameters to pass to the init system when - managing services. These will be passed as key=value - parameters to the init system's commandline. kwargs - are ignored for systemd enabled systems. - """ - return service('stop', service_name, **kwargs) - - -def service_restart(service_name, **kwargs): - """Restart a system service. - - The specified service name is managed via the system level init system. - Some init systems (e.g. upstart) require that additional arguments be - provided in order to directly control service instances whereas other init - systems allow for addressing instances of a service directly by name (e.g. - systemd). - - The kwargs allow for the additional parameters to be passed to underlying - init systems for those systems which require/allow for them. For example, - the ceph-osd upstart script requires the id parameter to be passed along - in order to identify which running daemon should be restarted. The follow- - ing example restarts the ceph-osd service for instance id=4: - - service_restart('ceph-osd', id=4) - - :param service_name: the name of the service to restart - :param **kwargs: additional parameters to pass to the init system when - managing services. These will be passed as key=value - parameters to the init system's commandline. kwargs - are ignored for init systems not allowing additional - parameters via the commandline (systemd). - """ - return service('restart', service_name) - - -def service_reload(service_name, restart_on_failure=False, **kwargs): - """Reload a system service, optionally falling back to restart if - reload fails. - - The specified service name is managed via the system level init system. - Some init systems (e.g. upstart) require that additional arguments be - provided in order to directly control service instances whereas other init - systems allow for addressing instances of a service directly by name (e.g. - systemd). - - The kwargs allow for the additional parameters to be passed to underlying - init systems for those systems which require/allow for them. For example, - the ceph-osd upstart script requires the id parameter to be passed along - in order to identify which running daemon should be reloaded. The follow- - ing example restarts the ceph-osd service for instance id=4: - - service_reload('ceph-osd', id=4) - - :param service_name: the name of the service to reload - :param restart_on_failure: boolean indicating whether to fallback to a - restart if the reload fails. - :param **kwargs: additional parameters to pass to the init system when - managing services. These will be passed as key=value - parameters to the init system's commandline. kwargs - are ignored for init systems not allowing additional - parameters via the commandline (systemd). - """ - service_result = service('reload', service_name, **kwargs) - if not service_result and restart_on_failure: - service_result = service('restart', service_name, **kwargs) - return service_result - - -def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d", - **kwargs): - """Pause a system service. - - Stop it, and prevent it from starting again at boot. - - :param service_name: the name of the service to pause - :param init_dir: path to the upstart init directory - :param initd_dir: path to the sysv init directory - :param **kwargs: additional parameters to pass to the init system when - managing services. These will be passed as key=value - parameters to the init system's commandline. kwargs - are ignored for init systems which do not support - key=value arguments via the commandline. - """ - stopped = True - if service_running(service_name, **kwargs): - stopped = service_stop(service_name, **kwargs) - upstart_file = os.path.join(init_dir, "{}.conf".format(service_name)) - sysv_file = os.path.join(initd_dir, service_name) - if init_is_systemd(): - service('disable', service_name) - service('mask', service_name) - elif os.path.exists(upstart_file): - override_path = os.path.join( - init_dir, '{}.override'.format(service_name)) - with open(override_path, 'w') as fh: - fh.write("manual\n") - elif os.path.exists(sysv_file): - subprocess.check_call(["update-rc.d", service_name, "disable"]) - else: - raise ValueError( - "Unable to detect {0} as SystemD, Upstart {1} or" - " SysV {2}".format( - service_name, upstart_file, sysv_file)) - return stopped - - -def service_resume(service_name, init_dir="/etc/init", - initd_dir="/etc/init.d", **kwargs): - """Resume a system service. - - Reenable starting again at boot. Start the service. - - :param service_name: the name of the service to resume - :param init_dir: the path to the init dir - :param initd dir: the path to the initd dir - :param **kwargs: additional parameters to pass to the init system when - managing services. These will be passed as key=value - parameters to the init system's commandline. kwargs - are ignored for systemd enabled systems. - """ - upstart_file = os.path.join(init_dir, "{}.conf".format(service_name)) - sysv_file = os.path.join(initd_dir, service_name) - if init_is_systemd(): - service('unmask', service_name) - service('enable', service_name) - elif os.path.exists(upstart_file): - override_path = os.path.join( - init_dir, '{}.override'.format(service_name)) - if os.path.exists(override_path): - os.unlink(override_path) - elif os.path.exists(sysv_file): - subprocess.check_call(["update-rc.d", service_name, "enable"]) - else: - raise ValueError( - "Unable to detect {0} as SystemD, Upstart {1} or" - " SysV {2}".format( - service_name, upstart_file, sysv_file)) - started = service_running(service_name, **kwargs) - - if not started: - started = service_start(service_name, **kwargs) - return started - - -def service(action, service_name, **kwargs): - """Control a system service. - - :param action: the action to take on the service - :param service_name: the name of the service to perform th action on - :param **kwargs: additional params to be passed to the service command in - the form of key=value. - """ - if init_is_systemd(): - cmd = ['systemctl', action, service_name] - else: - cmd = ['service', service_name, action] - for key, value in six.iteritems(kwargs): - parameter = '%s=%s' % (key, value) - cmd.append(parameter) - return subprocess.call(cmd) == 0 - - -_UPSTART_CONF = "/etc/init/{}.conf" -_INIT_D_CONF = "/etc/init.d/{}" - - -def service_running(service_name, **kwargs): - """Determine whether a system service is running. - - :param service_name: the name of the service - :param **kwargs: additional args to pass to the service command. This is - used to pass additional key=value arguments to the - service command line for managing specific instance - units (e.g. service ceph-osd status id=2). The kwargs - are ignored in systemd services. - """ - if init_is_systemd(): - return service('is-active', service_name) - else: - if os.path.exists(_UPSTART_CONF.format(service_name)): - try: - cmd = ['status', service_name] - for key, value in six.iteritems(kwargs): - parameter = '%s=%s' % (key, value) - cmd.append(parameter) - output = subprocess.check_output(cmd, - stderr=subprocess.STDOUT).decode('UTF-8') - except subprocess.CalledProcessError: - return False - else: - # This works for upstart scripts where the 'service' command - # returns a consistent string to represent running - # 'start/running' - if ("start/running" in output or - "is running" in output or - "up and running" in output): - return True - elif os.path.exists(_INIT_D_CONF.format(service_name)): - # Check System V scripts init script return codes - return service('status', service_name) - return False - - -SYSTEMD_SYSTEM = '/run/systemd/system' - - -def init_is_systemd(): - """Return True if the host system uses systemd, False otherwise.""" - if lsb_release()['DISTRIB_CODENAME'] == 'trusty': - return False - return os.path.isdir(SYSTEMD_SYSTEM) - - -def adduser(username, password=None, shell='/bin/bash', - system_user=False, primary_group=None, - secondary_groups=None, uid=None, home_dir=None): - """Add a user to the system. - - Will log but otherwise succeed if the user already exists. - - :param str username: Username to create - :param str password: Password for user; if ``None``, create a system user - :param str shell: The default shell for the user - :param bool system_user: Whether to create a login or system user - :param str primary_group: Primary group for user; defaults to username - :param list secondary_groups: Optional list of additional groups - :param int uid: UID for user being created - :param str home_dir: Home directory for user - - :returns: The password database entry struct, as returned by `pwd.getpwnam` - """ - try: - user_info = pwd.getpwnam(username) - log('user {0} already exists!'.format(username)) - if uid: - user_info = pwd.getpwuid(int(uid)) - log('user with uid {0} already exists!'.format(uid)) - except KeyError: - log('creating user {0}'.format(username)) - cmd = ['useradd'] - if uid: - cmd.extend(['--uid', str(uid)]) - if home_dir: - cmd.extend(['--home', str(home_dir)]) - if system_user or password is None: - cmd.append('--system') - else: - cmd.extend([ - '--create-home', - '--shell', shell, - '--password', password, - ]) - if not primary_group: - try: - grp.getgrnam(username) - primary_group = username # avoid "group exists" error - except KeyError: - pass - if primary_group: - cmd.extend(['-g', primary_group]) - if secondary_groups: - cmd.extend(['-G', ','.join(secondary_groups)]) - cmd.append(username) - subprocess.check_call(cmd) - user_info = pwd.getpwnam(username) - return user_info - - -def user_exists(username): - """Check if a user exists""" - try: - pwd.getpwnam(username) - user_exists = True - except KeyError: - user_exists = False - return user_exists - - -def uid_exists(uid): - """Check if a uid exists""" - try: - pwd.getpwuid(uid) - uid_exists = True - except KeyError: - uid_exists = False - return uid_exists - - -def group_exists(groupname): - """Check if a group exists""" - try: - grp.getgrnam(groupname) - group_exists = True - except KeyError: - group_exists = False - return group_exists - - -def gid_exists(gid): - """Check if a gid exists""" - try: - grp.getgrgid(gid) - gid_exists = True - except KeyError: - gid_exists = False - return gid_exists - - -def add_group(group_name, system_group=False, gid=None): - """Add a group to the system - - Will log but otherwise succeed if the group already exists. - - :param str group_name: group to create - :param bool system_group: Create system group - :param int gid: GID for user being created - - :returns: The password database entry struct, as returned by `grp.getgrnam` - """ - try: - group_info = grp.getgrnam(group_name) - log('group {0} already exists!'.format(group_name)) - if gid: - group_info = grp.getgrgid(gid) - log('group with gid {0} already exists!'.format(gid)) - except KeyError: - log('creating group {0}'.format(group_name)) - add_new_group(group_name, system_group, gid) - group_info = grp.getgrnam(group_name) - return group_info - - -def add_user_to_group(username, group): - """Add a user to a group""" - cmd = ['gpasswd', '-a', username, group] - log("Adding user {} to group {}".format(username, group)) - subprocess.check_call(cmd) - - -def chage(username, lastday=None, expiredate=None, inactive=None, - mindays=None, maxdays=None, root=None, warndays=None): - """Change user password expiry information - - :param str username: User to update - :param str lastday: Set when password was changed in YYYY-MM-DD format - :param str expiredate: Set when user's account will no longer be - accessible in YYYY-MM-DD format. - -1 will remove an account expiration date. - :param str inactive: Set the number of days of inactivity after a password - has expired before the account is locked. - -1 will remove an account's inactivity. - :param str mindays: Set the minimum number of days between password - changes to MIN_DAYS. - 0 indicates the password can be changed anytime. - :param str maxdays: Set the maximum number of days during which a - password is valid. - -1 as MAX_DAYS will remove checking maxdays - :param str root: Apply changes in the CHROOT_DIR directory - :param str warndays: Set the number of days of warning before a password - change is required - :raises subprocess.CalledProcessError: if call to chage fails - """ - cmd = ['chage'] - if root: - cmd.extend(['--root', root]) - if lastday: - cmd.extend(['--lastday', lastday]) - if expiredate: - cmd.extend(['--expiredate', expiredate]) - if inactive: - cmd.extend(['--inactive', inactive]) - if mindays: - cmd.extend(['--mindays', mindays]) - if maxdays: - cmd.extend(['--maxdays', maxdays]) - if warndays: - cmd.extend(['--warndays', warndays]) - cmd.append(username) - subprocess.check_call(cmd) - -remove_password_expiry = functools.partial(chage, expiredate='-1', inactive='-1', mindays='0', maxdays='-1') - -def rsync(from_path, to_path, flags='-r', options=None, timeout=None): - """Replicate the contents of a path""" - options = options or ['--delete', '--executability'] - cmd = ['/usr/bin/rsync', flags] - if timeout: - cmd = ['timeout', str(timeout)] + cmd - cmd.extend(options) - cmd.append(from_path) - cmd.append(to_path) - log(" ".join(cmd)) - return subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode('UTF-8').strip() - - -def symlink(source, destination): - """Create a symbolic link""" - log("Symlinking {} as {}".format(source, destination)) - cmd = [ - 'ln', - '-sf', - source, - destination, - ] - subprocess.check_call(cmd) - - -def mkdir(path, owner='root', group='root', perms=0o555, force=False): - """Create a directory""" - log("Making dir {} {}:{} {:o}".format(path, owner, group, - perms)) - uid = pwd.getpwnam(owner).pw_uid - gid = grp.getgrnam(group).gr_gid - realpath = os.path.abspath(path) - path_exists = os.path.exists(realpath) - if path_exists and force: - if not os.path.isdir(realpath): - log("Removing non-directory file {} prior to mkdir()".format(path)) - os.unlink(realpath) - os.makedirs(realpath, perms) - elif not path_exists: - os.makedirs(realpath, perms) - os.chown(realpath, uid, gid) - os.chmod(realpath, perms) - - -def write_file(path, content, owner='root', group='root', perms=0o444): - """Create or overwrite a file with the contents of a byte string.""" - uid = pwd.getpwnam(owner).pw_uid - gid = grp.getgrnam(group).gr_gid - # lets see if we can grab the file and compare the context, to avoid doing - # a write. - existing_content = None - existing_uid, existing_gid = None, None - try: - with open(path, 'rb') as target: - existing_content = target.read() - stat = os.stat(path) - existing_uid, existing_gid = stat.st_uid, stat.st_gid - except: - pass - if content != existing_content: - log("Writing file {} {}:{} {:o}".format(path, owner, group, perms), - level=DEBUG) - with open(path, 'wb') as target: - os.fchown(target.fileno(), uid, gid) - os.fchmod(target.fileno(), perms) - if six.PY3 and isinstance(content, six.string_types): - content = content.encode('UTF-8') - target.write(content) - return - # the contents were the same, but we might still need to change the - # ownership. - if existing_uid != uid: - log("Changing uid on already existing content: {} -> {}" - .format(existing_uid, uid), level=DEBUG) - os.chown(path, uid, -1) - if existing_gid != gid: - log("Changing gid on already existing content: {} -> {}" - .format(existing_gid, gid), level=DEBUG) - os.chown(path, -1, gid) - - -def fstab_remove(mp): - """Remove the given mountpoint entry from /etc/fstab""" - return Fstab.remove_by_mountpoint(mp) - - -def fstab_add(dev, mp, fs, options=None): - """Adds the given device entry to the /etc/fstab file""" - return Fstab.add(dev, mp, fs, options=options) - - -def mount(device, mountpoint, options=None, persist=False, filesystem="ext3"): - """Mount a filesystem at a particular mountpoint""" - cmd_args = ['mount'] - if options is not None: - cmd_args.extend(['-o', options]) - cmd_args.extend([device, mountpoint]) - try: - subprocess.check_output(cmd_args) - except subprocess.CalledProcessError as e: - log('Error mounting {} at {}\n{}'.format(device, mountpoint, e.output)) - return False - - if persist: - return fstab_add(device, mountpoint, filesystem, options=options) - return True - - -def umount(mountpoint, persist=False): - """Unmount a filesystem""" - cmd_args = ['umount', mountpoint] - try: - subprocess.check_output(cmd_args) - except subprocess.CalledProcessError as e: - log('Error unmounting {}\n{}'.format(mountpoint, e.output)) - return False - - if persist: - return fstab_remove(mountpoint) - return True - - -def mounts(): - """Get a list of all mounted volumes as [[mountpoint,device],[...]]""" - with open('/proc/mounts') as f: - # [['/mount/point','/dev/path'],[...]] - system_mounts = [m[1::-1] for m in [l.strip().split() - for l in f.readlines()]] - return system_mounts - - -def fstab_mount(mountpoint): - """Mount filesystem using fstab""" - cmd_args = ['mount', mountpoint] - try: - subprocess.check_output(cmd_args) - except subprocess.CalledProcessError as e: - log('Error unmounting {}\n{}'.format(mountpoint, e.output)) - return False - return True - - -def file_hash(path, hash_type='md5'): - """Generate a hash checksum of the contents of 'path' or None if not found. - - :param str hash_type: Any hash alrgorithm supported by :mod:`hashlib`, - such as md5, sha1, sha256, sha512, etc. - """ - if os.path.exists(path): - h = getattr(hashlib, hash_type)() - with open(path, 'rb') as source: - h.update(source.read()) - return h.hexdigest() - else: - return None - - -def path_hash(path): - """Generate a hash checksum of all files matching 'path'. Standard - wildcards like '*' and '?' are supported, see documentation for the 'glob' - module for more information. - - :return: dict: A { filename: hash } dictionary for all matched files. - Empty if none found. - """ - return { - filename: file_hash(filename) - for filename in glob.iglob(path) - } - - -def check_hash(path, checksum, hash_type='md5'): - """Validate a file using a cryptographic checksum. - - :param str checksum: Value of the checksum used to validate the file. - :param str hash_type: Hash algorithm used to generate `checksum`. - Can be any hash alrgorithm supported by :mod:`hashlib`, - such as md5, sha1, sha256, sha512, etc. - :raises ChecksumError: If the file fails the checksum - - """ - actual_checksum = file_hash(path, hash_type) - if checksum != actual_checksum: - raise ChecksumError("'%s' != '%s'" % (checksum, actual_checksum)) - - -class ChecksumError(ValueError): - """A class derived from Value error to indicate the checksum failed.""" - pass - - -def restart_on_change(restart_map, stopstart=False, restart_functions=None): - """Restart services based on configuration files changing - - This function is used a decorator, for example:: - - @restart_on_change({ - '/etc/ceph/ceph.conf': [ 'cinder-api', 'cinder-volume' ] - '/etc/apache/sites-enabled/*': [ 'apache2' ] - }) - def config_changed(): - pass # your code here - - In this example, the cinder-api and cinder-volume services - would be restarted if /etc/ceph/ceph.conf is changed by the - ceph_client_changed function. The apache2 service would be - restarted if any file matching the pattern got changed, created - or removed. Standard wildcards are supported, see documentation - for the 'glob' module for more information. - - @param restart_map: {path_file_name: [service_name, ...] - @param stopstart: DEFAULT false; whether to stop, start OR restart - @param restart_functions: nonstandard functions to use to restart services - {svc: func, ...} - @returns result from decorated function - """ - def wrap(f): - @functools.wraps(f) - def wrapped_f(*args, **kwargs): - return restart_on_change_helper( - (lambda: f(*args, **kwargs)), restart_map, stopstart, - restart_functions) - return wrapped_f - return wrap - - -def restart_on_change_helper(lambda_f, restart_map, stopstart=False, - restart_functions=None): - """Helper function to perform the restart_on_change function. - - This is provided for decorators to restart services if files described - in the restart_map have changed after an invocation of lambda_f(). - - @param lambda_f: function to call. - @param restart_map: {file: [service, ...]} - @param stopstart: whether to stop, start or restart a service - @param restart_functions: nonstandard functions to use to restart services - {svc: func, ...} - @returns result of lambda_f() - """ - if restart_functions is None: - restart_functions = {} - checksums = {path: path_hash(path) for path in restart_map} - r = lambda_f() - # create a list of lists of the services to restart - restarts = [restart_map[path] - for path in restart_map - if path_hash(path) != checksums[path]] - # create a flat list of ordered services without duplicates from lists - services_list = list(OrderedDict.fromkeys(itertools.chain(*restarts))) - if services_list: - actions = ('stop', 'start') if stopstart else ('restart',) - for service_name in services_list: - if service_name in restart_functions: - restart_functions[service_name](service_name) - else: - for action in actions: - service(action, service_name) - return r - - -def pwgen(length=None): - """Generate a random pasword.""" - if length is None: - # A random length is ok to use a weak PRNG - length = random.choice(range(35, 45)) - alphanumeric_chars = [ - l for l in (string.ascii_letters + string.digits) - if l not in 'l0QD1vAEIOUaeiou'] - # Use a crypto-friendly PRNG (e.g. /dev/urandom) for making the - # actual password - random_generator = random.SystemRandom() - random_chars = [ - random_generator.choice(alphanumeric_chars) for _ in range(length)] - return(''.join(random_chars)) - - -def is_phy_iface(interface): - """Returns True if interface is not virtual, otherwise False.""" - if interface: - sys_net = '/sys/class/net' - if os.path.isdir(sys_net): - for iface in glob.glob(os.path.join(sys_net, '*')): - if '/virtual/' in os.path.realpath(iface): - continue - - if interface == os.path.basename(iface): - return True - - return False - - -def get_bond_master(interface): - """Returns bond master if interface is bond slave otherwise None. - - NOTE: the provided interface is expected to be physical - """ - if interface: - iface_path = '/sys/class/net/%s' % (interface) - if os.path.exists(iface_path): - if '/virtual/' in os.path.realpath(iface_path): - return None - - master = os.path.join(iface_path, 'master') - if os.path.exists(master): - master = os.path.realpath(master) - # make sure it is a bond master - if os.path.exists(os.path.join(master, 'bonding')): - return os.path.basename(master) - - return None - - -def list_nics(nic_type=None): - """Return a list of nics of given type(s)""" - if isinstance(nic_type, six.string_types): - int_types = [nic_type] - else: - int_types = nic_type - - interfaces = [] - if nic_type: - for int_type in int_types: - cmd = ['ip', 'addr', 'show', 'label', int_type + '*'] - ip_output = subprocess.check_output(cmd).decode('UTF-8') - ip_output = ip_output.split('\n') - ip_output = (line for line in ip_output if line) - for line in ip_output: - if line.split()[1].startswith(int_type): - matched = re.search('.*: (' + int_type + - r'[0-9]+\.[0-9]+)@.*', line) - if matched: - iface = matched.groups()[0] - else: - iface = line.split()[1].replace(":", "") - - if iface not in interfaces: - interfaces.append(iface) - else: - cmd = ['ip', 'a'] - ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n') - ip_output = (line.strip() for line in ip_output if line) - - key = re.compile('^[0-9]+:\s+(.+):') - for line in ip_output: - matched = re.search(key, line) - if matched: - iface = matched.group(1) - iface = iface.partition("@")[0] - if iface not in interfaces: - interfaces.append(iface) - - return interfaces - - -def set_nic_mtu(nic, mtu): - """Set the Maximum Transmission Unit (MTU) on a network interface.""" - cmd = ['ip', 'link', 'set', nic, 'mtu', mtu] - subprocess.check_call(cmd) - - -def get_nic_mtu(nic): - """Return the Maximum Transmission Unit (MTU) for a network interface.""" - cmd = ['ip', 'addr', 'show', nic] - ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n') - mtu = "" - for line in ip_output: - words = line.split() - if 'mtu' in words: - mtu = words[words.index("mtu") + 1] - return mtu - - -def get_nic_hwaddr(nic): - """Return the Media Access Control (MAC) for a network interface.""" - cmd = ['ip', '-o', '-0', 'addr', 'show', nic] - ip_output = subprocess.check_output(cmd).decode('UTF-8') - hwaddr = "" - words = ip_output.split() - if 'link/ether' in words: - hwaddr = words[words.index('link/ether') + 1] - return hwaddr - - -@contextmanager -def chdir(directory): - """Change the current working directory to a different directory for a code - block and return the previous directory after the block exits. Useful to - run commands from a specificed directory. - - :param str directory: The directory path to change to for this context. - """ - cur = os.getcwd() - try: - yield os.chdir(directory) - finally: - os.chdir(cur) - - -def chownr(path, owner, group, follow_links=True, chowntopdir=False): - """Recursively change user and group ownership of files and directories - in given path. Doesn't chown path itself by default, only its children. - - :param str path: The string path to start changing ownership. - :param str owner: The owner string to use when looking up the uid. - :param str group: The group string to use when looking up the gid. - :param bool follow_links: Also follow and chown links if True - :param bool chowntopdir: Also chown path itself if True - """ - uid = pwd.getpwnam(owner).pw_uid - gid = grp.getgrnam(group).gr_gid - if follow_links: - chown = os.chown - else: - chown = os.lchown - - if chowntopdir: - broken_symlink = os.path.lexists(path) and not os.path.exists(path) - if not broken_symlink: - chown(path, uid, gid) - for root, dirs, files in os.walk(path, followlinks=follow_links): - for name in dirs + files: - full = os.path.join(root, name) - broken_symlink = os.path.lexists(full) and not os.path.exists(full) - if not broken_symlink: - chown(full, uid, gid) - - -def lchownr(path, owner, group): - """Recursively change user and group ownership of files and directories - in a given path, not following symbolic links. See the documentation for - 'os.lchown' for more information. - - :param str path: The string path to start changing ownership. - :param str owner: The owner string to use when looking up the uid. - :param str group: The group string to use when looking up the gid. - """ - chownr(path, owner, group, follow_links=False) - - -def owner(path): - """Returns a tuple containing the username & groupname owning the path. - - :param str path: the string path to retrieve the ownership - :return tuple(str, str): A (username, groupname) tuple containing the - name of the user and group owning the path. - :raises OSError: if the specified path does not exist - """ - stat = os.stat(path) - username = pwd.getpwuid(stat.st_uid)[0] - groupname = grp.getgrgid(stat.st_gid)[0] - return username, groupname - - -def get_total_ram(): - """The total amount of system RAM in bytes. - - This is what is reported by the OS, and may be overcommitted when - there are multiple containers hosted on the same machine. - """ - with open('/proc/meminfo', 'r') as f: - for line in f.readlines(): - if line: - key, value, unit = line.split() - if key == 'MemTotal:': - assert unit == 'kB', 'Unknown unit' - return int(value) * 1024 # Classic, not KiB. - raise NotImplementedError() - - -UPSTART_CONTAINER_TYPE = '/run/container_type' - - -def is_container(): - """Determine whether unit is running in a container - - @return: boolean indicating if unit is in a container - """ - if init_is_systemd(): - # Detect using systemd-detect-virt - return subprocess.call(['systemd-detect-virt', - '--container']) == 0 - else: - # Detect using upstart container file marker - return os.path.exists(UPSTART_CONTAINER_TYPE) - - -def add_to_updatedb_prunepath(path, updatedb_path=UPDATEDB_PATH): - with open(updatedb_path, 'r+') as f_id: - updatedb_text = f_id.read() - output = updatedb(updatedb_text, path) - f_id.seek(0) - f_id.write(output) - f_id.truncate() - - -def updatedb(updatedb_text, new_path): - lines = [line for line in updatedb_text.split("\n")] - for i, line in enumerate(lines): - if line.startswith("PRUNEPATHS="): - paths_line = line.split("=")[1].replace('"', '') - paths = paths_line.split(" ") - if new_path not in paths: - paths.append(new_path) - lines[i] = 'PRUNEPATHS="{}"'.format(' '.join(paths)) - output = "\n".join(lines) - return output - - -def modulo_distribution(modulo=3, wait=30): - """ Modulo distribution - - This helper uses the unit number, a modulo value and a constant wait time - to produce a calculated wait time distribution. This is useful in large - scale deployments to distribute load during an expensive operation such as - service restarts. - - If you have 1000 nodes that need to restart 100 at a time 1 minute at a - time: - - time.wait(modulo_distribution(modulo=100, wait=60)) - restart() - - If you need restarts to happen serially set modulo to the exact number of - nodes and set a high constant wait time: - - time.wait(modulo_distribution(modulo=10, wait=120)) - restart() - - @param modulo: int The modulo number creates the group distribution - @param wait: int The constant time wait value - @return: int Calculated time to wait for unit operation - """ - unit_number = int(local_unit().split('/')[1]) - return (unit_number % modulo) * wait diff --git a/hooks/charmhelpers/core/host_factory/__init__.py b/hooks/charmhelpers/core/host_factory/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/hooks/charmhelpers/core/host_factory/centos.py b/hooks/charmhelpers/core/host_factory/centos.py deleted file mode 100644 index 7781a39..0000000 --- a/hooks/charmhelpers/core/host_factory/centos.py +++ /dev/null @@ -1,72 +0,0 @@ -import subprocess -import yum -import os - -from charmhelpers.core.strutils import BasicStringComparator - - -class CompareHostReleases(BasicStringComparator): - """Provide comparisons of Host releases. - - Use in the form of - - if CompareHostReleases(release) > 'trusty': - # do something with mitaka - """ - - def __init__(self, item): - raise NotImplementedError( - "CompareHostReleases() is not implemented for CentOS") - - -def service_available(service_name): - # """Determine whether a system service is available.""" - if os.path.isdir('/run/systemd/system'): - cmd = ['systemctl', 'is-enabled', service_name] - else: - cmd = ['service', service_name, 'is-enabled'] - return subprocess.call(cmd) == 0 - - -def add_new_group(group_name, system_group=False, gid=None): - cmd = ['groupadd'] - if gid: - cmd.extend(['--gid', str(gid)]) - if system_group: - cmd.append('-r') - cmd.append(group_name) - subprocess.check_call(cmd) - - -def lsb_release(): - """Return /etc/os-release in a dict.""" - d = {} - with open('/etc/os-release', 'r') as lsb: - for l in lsb: - s = l.split('=') - if len(s) != 2: - continue - d[s[0].strip()] = s[1].strip() - return d - - -def cmp_pkgrevno(package, revno, pkgcache=None): - """Compare supplied revno with the revno of the installed package. - - * 1 => Installed revno is greater than supplied arg - * 0 => Installed revno is the same as supplied arg - * -1 => Installed revno is less than supplied arg - - This function imports YumBase function if the pkgcache argument - is None. - """ - if not pkgcache: - y = yum.YumBase() - packages = y.doPackageLists() - pkgcache = {i.Name: i.version for i in packages['installed']} - pkg = pkgcache[package] - if pkg > revno: - return 1 - if pkg < revno: - return -1 - return 0 diff --git a/hooks/charmhelpers/core/host_factory/ubuntu.py b/hooks/charmhelpers/core/host_factory/ubuntu.py deleted file mode 100644 index 99451b5..0000000 --- a/hooks/charmhelpers/core/host_factory/ubuntu.py +++ /dev/null @@ -1,90 +0,0 @@ -import subprocess - -from charmhelpers.core.strutils import BasicStringComparator - - -UBUNTU_RELEASES = ( - 'lucid', - 'maverick', - 'natty', - 'oneiric', - 'precise', - 'quantal', - 'raring', - 'saucy', - 'trusty', - 'utopic', - 'vivid', - 'wily', - 'xenial', - 'yakkety', - 'zesty', - 'artful', - 'bionic', -) - - -class CompareHostReleases(BasicStringComparator): - """Provide comparisons of Ubuntu releases. - - Use in the form of - - if CompareHostReleases(release) > 'trusty': - # do something with mitaka - """ - _list = UBUNTU_RELEASES - - -def service_available(service_name): - """Determine whether a system service is available""" - try: - subprocess.check_output( - ['service', service_name, 'status'], - stderr=subprocess.STDOUT).decode('UTF-8') - except subprocess.CalledProcessError as e: - return b'unrecognized service' not in e.output - else: - return True - - -def add_new_group(group_name, system_group=False, gid=None): - cmd = ['addgroup'] - if gid: - cmd.extend(['--gid', str(gid)]) - if system_group: - cmd.append('--system') - else: - cmd.extend([ - '--group', - ]) - cmd.append(group_name) - subprocess.check_call(cmd) - - -def lsb_release(): - """Return /etc/lsb-release in a dict""" - d = {} - with open('/etc/lsb-release', 'r') as lsb: - for l in lsb: - k, v = l.split('=') - d[k.strip()] = v.strip() - return d - - -def cmp_pkgrevno(package, revno, pkgcache=None): - """Compare supplied revno with the revno of the installed package. - - * 1 => Installed revno is greater than supplied arg - * 0 => Installed revno is the same as supplied arg - * -1 => Installed revno is less than supplied arg - - This function imports apt_cache function from charmhelpers.fetch if - the pkgcache argument is None. Be sure to add charmhelpers.fetch if - you call this function, or pass an apt_pkg.Cache() instance. - """ - import apt_pkg - if not pkgcache: - from charmhelpers.fetch import apt_cache - pkgcache = apt_cache() - pkg = pkgcache[package] - return apt_pkg.version_compare(pkg.current_ver.ver_str, revno) diff --git a/hooks/charmhelpers/core/hugepage.py b/hooks/charmhelpers/core/hugepage.py deleted file mode 100644 index 54b5b5e..0000000 --- a/hooks/charmhelpers/core/hugepage.py +++ /dev/null @@ -1,69 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import yaml -from charmhelpers.core import fstab -from charmhelpers.core import sysctl -from charmhelpers.core.host import ( - add_group, - add_user_to_group, - fstab_mount, - mkdir, -) -from charmhelpers.core.strutils import bytes_from_string -from subprocess import check_output - - -def hugepage_support(user, group='hugetlb', nr_hugepages=256, - max_map_count=65536, mnt_point='/run/hugepages/kvm', - pagesize='2MB', mount=True, set_shmmax=False): - """Enable hugepages on system. - - Args: - user (str) -- Username to allow access to hugepages to - group (str) -- Group name to own hugepages - nr_hugepages (int) -- Number of pages to reserve - max_map_count (int) -- Number of Virtual Memory Areas a process can own - mnt_point (str) -- Directory to mount hugepages on - pagesize (str) -- Size of hugepages - mount (bool) -- Whether to Mount hugepages - """ - group_info = add_group(group) - gid = group_info.gr_gid - add_user_to_group(user, group) - if max_map_count < 2 * nr_hugepages: - max_map_count = 2 * nr_hugepages - sysctl_settings = { - 'vm.nr_hugepages': nr_hugepages, - 'vm.max_map_count': max_map_count, - 'vm.hugetlb_shm_group': gid, - } - if set_shmmax: - shmmax_current = int(check_output(['sysctl', '-n', 'kernel.shmmax'])) - shmmax_minsize = bytes_from_string(pagesize) * nr_hugepages - if shmmax_minsize > shmmax_current: - sysctl_settings['kernel.shmmax'] = shmmax_minsize - sysctl.create(yaml.dump(sysctl_settings), '/etc/sysctl.d/10-hugepage.conf') - mkdir(mnt_point, owner='root', group='root', perms=0o755, force=False) - lfstab = fstab.Fstab() - fstab_entry = lfstab.get_entry_by_attr('mountpoint', mnt_point) - if fstab_entry: - lfstab.remove_entry(fstab_entry) - entry = lfstab.Entry('nodev', mnt_point, 'hugetlbfs', - 'mode=1770,gid={},pagesize={}'.format(gid, pagesize), 0, 0) - lfstab.add_entry(entry) - if mount: - fstab_mount(mnt_point) diff --git a/hooks/charmhelpers/core/kernel.py b/hooks/charmhelpers/core/kernel.py deleted file mode 100644 index 2d40452..0000000 --- a/hooks/charmhelpers/core/kernel.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -import subprocess - -from charmhelpers.osplatform import get_platform -from charmhelpers.core.hookenv import ( - log, - INFO -) - -__platform__ = get_platform() -if __platform__ == "ubuntu": - from charmhelpers.core.kernel_factory.ubuntu import ( - persistent_modprobe, - update_initramfs, - ) # flake8: noqa -- ignore F401 for this import -elif __platform__ == "centos": - from charmhelpers.core.kernel_factory.centos import ( - persistent_modprobe, - update_initramfs, - ) # flake8: noqa -- ignore F401 for this import - -__author__ = "Jorge Niedbalski " - - -def modprobe(module, persist=True): - """Load a kernel module and configure for auto-load on reboot.""" - cmd = ['modprobe', module] - - log('Loading kernel module %s' % module, level=INFO) - - subprocess.check_call(cmd) - if persist: - persistent_modprobe(module) - - -def rmmod(module, force=False): - """Remove a module from the linux kernel""" - cmd = ['rmmod'] - if force: - cmd.append('-f') - cmd.append(module) - log('Removing kernel module %s' % module, level=INFO) - return subprocess.check_call(cmd) - - -def lsmod(): - """Shows what kernel modules are currently loaded""" - return subprocess.check_output(['lsmod'], - universal_newlines=True) - - -def is_module_loaded(module): - """Checks if a kernel module is already loaded""" - matches = re.findall('^%s[ ]+' % module, lsmod(), re.M) - return len(matches) > 0 diff --git a/hooks/charmhelpers/core/kernel_factory/__init__.py b/hooks/charmhelpers/core/kernel_factory/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/hooks/charmhelpers/core/kernel_factory/centos.py b/hooks/charmhelpers/core/kernel_factory/centos.py deleted file mode 100644 index 1c402c1..0000000 --- a/hooks/charmhelpers/core/kernel_factory/centos.py +++ /dev/null @@ -1,17 +0,0 @@ -import subprocess -import os - - -def persistent_modprobe(module): - """Load a kernel module and configure for auto-load on reboot.""" - if not os.path.exists('/etc/rc.modules'): - open('/etc/rc.modules', 'a') - os.chmod('/etc/rc.modules', 111) - with open('/etc/rc.modules', 'r+') as modules: - if module not in modules.read(): - modules.write('modprobe %s\n' % module) - - -def update_initramfs(version='all'): - """Updates an initramfs image.""" - return subprocess.check_call(["dracut", "-f", version]) diff --git a/hooks/charmhelpers/core/kernel_factory/ubuntu.py b/hooks/charmhelpers/core/kernel_factory/ubuntu.py deleted file mode 100644 index 3de372f..0000000 --- a/hooks/charmhelpers/core/kernel_factory/ubuntu.py +++ /dev/null @@ -1,13 +0,0 @@ -import subprocess - - -def persistent_modprobe(module): - """Load a kernel module and configure for auto-load on reboot.""" - with open('/etc/modules', 'r+') as modules: - if module not in modules.read(): - modules.write(module + "\n") - - -def update_initramfs(version='all'): - """Updates an initramfs image.""" - return subprocess.check_call(["update-initramfs", "-k", version, "-u"]) diff --git a/hooks/charmhelpers/core/services/__init__.py b/hooks/charmhelpers/core/services/__init__.py deleted file mode 100644 index 61fd074..0000000 --- a/hooks/charmhelpers/core/services/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .base import * # NOQA -from .helpers import * # NOQA diff --git a/hooks/charmhelpers/core/services/base.py b/hooks/charmhelpers/core/services/base.py deleted file mode 100644 index ca9dc99..0000000 --- a/hooks/charmhelpers/core/services/base.py +++ /dev/null @@ -1,351 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import json -from inspect import getargspec -from collections import Iterable, OrderedDict - -from charmhelpers.core import host -from charmhelpers.core import hookenv - - -__all__ = ['ServiceManager', 'ManagerCallback', - 'PortManagerCallback', 'open_ports', 'close_ports', 'manage_ports', - 'service_restart', 'service_stop'] - - -class ServiceManager(object): - def __init__(self, services=None): - """ - Register a list of services, given their definitions. - - Service definitions are dicts in the following formats (all keys except - 'service' are optional):: - - { - "service": , - "required_data": , - "provided_data": , - "data_ready": , - "data_lost": , - "start": , - "stop": , - "ports": , - } - - The 'required_data' list should contain dicts of required data (or - dependency managers that act like dicts and know how to collect the data). - Only when all items in the 'required_data' list are populated are the list - of 'data_ready' and 'start' callbacks executed. See `is_ready()` for more - information. - - The 'provided_data' list should contain relation data providers, most likely - a subclass of :class:`charmhelpers.core.services.helpers.RelationContext`, - that will indicate a set of data to set on a given relation. - - The 'data_ready' value should be either a single callback, or a list of - callbacks, to be called when all items in 'required_data' pass `is_ready()`. - Each callback will be called with the service name as the only parameter. - After all of the 'data_ready' callbacks are called, the 'start' callbacks - are fired. - - The 'data_lost' value should be either a single callback, or a list of - callbacks, to be called when a 'required_data' item no longer passes - `is_ready()`. Each callback will be called with the service name as the - only parameter. After all of the 'data_lost' callbacks are called, - the 'stop' callbacks are fired. - - The 'start' value should be either a single callback, or a list of - callbacks, to be called when starting the service, after the 'data_ready' - callbacks are complete. Each callback will be called with the service - name as the only parameter. This defaults to - `[host.service_start, services.open_ports]`. - - The 'stop' value should be either a single callback, or a list of - callbacks, to be called when stopping the service. If the service is - being stopped because it no longer has all of its 'required_data', this - will be called after all of the 'data_lost' callbacks are complete. - Each callback will be called with the service name as the only parameter. - This defaults to `[services.close_ports, host.service_stop]`. - - The 'ports' value should be a list of ports to manage. The default - 'start' handler will open the ports after the service is started, - and the default 'stop' handler will close the ports prior to stopping - the service. - - - Examples: - - The following registers an Upstart service called bingod that depends on - a mongodb relation and which runs a custom `db_migrate` function prior to - restarting the service, and a Runit service called spadesd:: - - manager = services.ServiceManager([ - { - 'service': 'bingod', - 'ports': [80, 443], - 'required_data': [MongoRelation(), config(), {'my': 'data'}], - 'data_ready': [ - services.template(source='bingod.conf'), - services.template(source='bingod.ini', - target='/etc/bingod.ini', - owner='bingo', perms=0400), - ], - }, - { - 'service': 'spadesd', - 'data_ready': services.template(source='spadesd_run.j2', - target='/etc/sv/spadesd/run', - perms=0555), - 'start': runit_start, - 'stop': runit_stop, - }, - ]) - manager.manage() - """ - self._ready_file = os.path.join(hookenv.charm_dir(), 'READY-SERVICES.json') - self._ready = None - self.services = OrderedDict() - for service in services or []: - service_name = service['service'] - self.services[service_name] = service - - def manage(self): - """ - Handle the current hook by doing The Right Thing with the registered services. - """ - hookenv._run_atstart() - try: - hook_name = hookenv.hook_name() - if hook_name == 'stop': - self.stop_services() - else: - self.reconfigure_services() - self.provide_data() - except SystemExit as x: - if x.code is None or x.code == 0: - hookenv._run_atexit() - hookenv._run_atexit() - - def provide_data(self): - """ - Set the relation data for each provider in the ``provided_data`` list. - - A provider must have a `name` attribute, which indicates which relation - to set data on, and a `provide_data()` method, which returns a dict of - data to set. - - The `provide_data()` method can optionally accept two parameters: - - * ``remote_service`` The name of the remote service that the data will - be provided to. The `provide_data()` method will be called once - for each connected service (not unit). This allows the method to - tailor its data to the given service. - * ``service_ready`` Whether or not the service definition had all of - its requirements met, and thus the ``data_ready`` callbacks run. - - Note that the ``provided_data`` methods are now called **after** the - ``data_ready`` callbacks are run. This gives the ``data_ready`` callbacks - a chance to generate any data necessary for the providing to the remote - services. - """ - for service_name, service in self.services.items(): - service_ready = self.is_ready(service_name) - for provider in service.get('provided_data', []): - for relid in hookenv.relation_ids(provider.name): - units = hookenv.related_units(relid) - if not units: - continue - remote_service = units[0].split('/')[0] - argspec = getargspec(provider.provide_data) - if len(argspec.args) > 1: - data = provider.provide_data(remote_service, service_ready) - else: - data = provider.provide_data() - if data: - hookenv.relation_set(relid, data) - - def reconfigure_services(self, *service_names): - """ - Update all files for one or more registered services, and, - if ready, optionally restart them. - - If no service names are given, reconfigures all registered services. - """ - for service_name in service_names or self.services.keys(): - if self.is_ready(service_name): - self.fire_event('data_ready', service_name) - self.fire_event('start', service_name, default=[ - service_restart, - manage_ports]) - self.save_ready(service_name) - else: - if self.was_ready(service_name): - self.fire_event('data_lost', service_name) - self.fire_event('stop', service_name, default=[ - manage_ports, - service_stop]) - self.save_lost(service_name) - - def stop_services(self, *service_names): - """ - Stop one or more registered services, by name. - - If no service names are given, stops all registered services. - """ - for service_name in service_names or self.services.keys(): - self.fire_event('stop', service_name, default=[ - manage_ports, - service_stop]) - - def get_service(self, service_name): - """ - Given the name of a registered service, return its service definition. - """ - service = self.services.get(service_name) - if not service: - raise KeyError('Service not registered: %s' % service_name) - return service - - def fire_event(self, event_name, service_name, default=None): - """ - Fire a data_ready, data_lost, start, or stop event on a given service. - """ - service = self.get_service(service_name) - callbacks = service.get(event_name, default) - if not callbacks: - return - if not isinstance(callbacks, Iterable): - callbacks = [callbacks] - for callback in callbacks: - if isinstance(callback, ManagerCallback): - callback(self, service_name, event_name) - else: - callback(service_name) - - def is_ready(self, service_name): - """ - Determine if a registered service is ready, by checking its 'required_data'. - - A 'required_data' item can be any mapping type, and is considered ready - if `bool(item)` evaluates as True. - """ - service = self.get_service(service_name) - reqs = service.get('required_data', []) - return all(bool(req) for req in reqs) - - def _load_ready_file(self): - if self._ready is not None: - return - if os.path.exists(self._ready_file): - with open(self._ready_file) as fp: - self._ready = set(json.load(fp)) - else: - self._ready = set() - - def _save_ready_file(self): - if self._ready is None: - return - with open(self._ready_file, 'w') as fp: - json.dump(list(self._ready), fp) - - def save_ready(self, service_name): - """ - Save an indicator that the given service is now data_ready. - """ - self._load_ready_file() - self._ready.add(service_name) - self._save_ready_file() - - def save_lost(self, service_name): - """ - Save an indicator that the given service is no longer data_ready. - """ - self._load_ready_file() - self._ready.discard(service_name) - self._save_ready_file() - - def was_ready(self, service_name): - """ - Determine if the given service was previously data_ready. - """ - self._load_ready_file() - return service_name in self._ready - - -class ManagerCallback(object): - """ - Special case of a callback that takes the `ServiceManager` instance - in addition to the service name. - - Subclasses should implement `__call__` which should accept three parameters: - - * `manager` The `ServiceManager` instance - * `service_name` The name of the service it's being triggered for - * `event_name` The name of the event that this callback is handling - """ - def __call__(self, manager, service_name, event_name): - raise NotImplementedError() - - -class PortManagerCallback(ManagerCallback): - """ - Callback class that will open or close ports, for use as either - a start or stop action. - """ - def __call__(self, manager, service_name, event_name): - service = manager.get_service(service_name) - new_ports = service.get('ports', []) - port_file = os.path.join(hookenv.charm_dir(), '.{}.ports'.format(service_name)) - if os.path.exists(port_file): - with open(port_file) as fp: - old_ports = fp.read().split(',') - for old_port in old_ports: - if bool(old_port): - old_port = int(old_port) - if old_port not in new_ports: - hookenv.close_port(old_port) - with open(port_file, 'w') as fp: - fp.write(','.join(str(port) for port in new_ports)) - for port in new_ports: - if event_name == 'start': - hookenv.open_port(port) - elif event_name == 'stop': - hookenv.close_port(port) - - -def service_stop(service_name): - """ - Wrapper around host.service_stop to prevent spurious "unknown service" - messages in the logs. - """ - if host.service_running(service_name): - host.service_stop(service_name) - - -def service_restart(service_name): - """ - Wrapper around host.service_restart to prevent spurious "unknown service" - messages in the logs. - """ - if host.service_available(service_name): - if host.service_running(service_name): - host.service_restart(service_name) - else: - host.service_start(service_name) - - -# Convenience aliases -open_ports = close_ports = manage_ports = PortManagerCallback() diff --git a/hooks/charmhelpers/core/services/helpers.py b/hooks/charmhelpers/core/services/helpers.py deleted file mode 100644 index 3e6e30d..0000000 --- a/hooks/charmhelpers/core/services/helpers.py +++ /dev/null @@ -1,290 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import yaml - -from charmhelpers.core import hookenv -from charmhelpers.core import host -from charmhelpers.core import templating - -from charmhelpers.core.services.base import ManagerCallback - - -__all__ = ['RelationContext', 'TemplateCallback', - 'render_template', 'template'] - - -class RelationContext(dict): - """ - Base class for a context generator that gets relation data from juju. - - Subclasses must provide the attributes `name`, which is the name of the - interface of interest, `interface`, which is the type of the interface of - interest, and `required_keys`, which is the set of keys required for the - relation to be considered complete. The data for all interfaces matching - the `name` attribute that are complete will used to populate the dictionary - values (see `get_data`, below). - - The generated context will be namespaced under the relation :attr:`name`, - to prevent potential naming conflicts. - - :param str name: Override the relation :attr:`name`, since it can vary from charm to charm - :param list additional_required_keys: Extend the list of :attr:`required_keys` - """ - name = None - interface = None - - def __init__(self, name=None, additional_required_keys=None): - if not hasattr(self, 'required_keys'): - self.required_keys = [] - - if name is not None: - self.name = name - if additional_required_keys: - self.required_keys.extend(additional_required_keys) - self.get_data() - - def __bool__(self): - """ - Returns True if all of the required_keys are available. - """ - return self.is_ready() - - __nonzero__ = __bool__ - - def __repr__(self): - return super(RelationContext, self).__repr__() - - def is_ready(self): - """ - Returns True if all of the `required_keys` are available from any units. - """ - ready = len(self.get(self.name, [])) > 0 - if not ready: - hookenv.log('Incomplete relation: {}'.format(self.__class__.__name__), hookenv.DEBUG) - return ready - - def _is_ready(self, unit_data): - """ - Helper method that tests a set of relation data and returns True if - all of the `required_keys` are present. - """ - return set(unit_data.keys()).issuperset(set(self.required_keys)) - - def get_data(self): - """ - Retrieve the relation data for each unit involved in a relation and, - if complete, store it in a list under `self[self.name]`. This - is automatically called when the RelationContext is instantiated. - - The units are sorted lexographically first by the service ID, then by - the unit ID. Thus, if an interface has two other services, 'db:1' - and 'db:2', with 'db:1' having two units, 'wordpress/0' and 'wordpress/1', - and 'db:2' having one unit, 'mediawiki/0', all of which have a complete - set of data, the relation data for the units will be stored in the - order: 'wordpress/0', 'wordpress/1', 'mediawiki/0'. - - If you only care about a single unit on the relation, you can just - access it as `{{ interface[0]['key'] }}`. However, if you can at all - support multiple units on a relation, you should iterate over the list, - like:: - - {% for unit in interface -%} - {{ unit['key'] }}{% if not loop.last %},{% endif %} - {%- endfor %} - - Note that since all sets of relation data from all related services and - units are in a single list, if you need to know which service or unit a - set of data came from, you'll need to extend this class to preserve - that information. - """ - if not hookenv.relation_ids(self.name): - return - - ns = self.setdefault(self.name, []) - for rid in sorted(hookenv.relation_ids(self.name)): - for unit in sorted(hookenv.related_units(rid)): - reldata = hookenv.relation_get(rid=rid, unit=unit) - if self._is_ready(reldata): - ns.append(reldata) - - def provide_data(self): - """ - Return data to be relation_set for this interface. - """ - return {} - - -class MysqlRelation(RelationContext): - """ - Relation context for the `mysql` interface. - - :param str name: Override the relation :attr:`name`, since it can vary from charm to charm - :param list additional_required_keys: Extend the list of :attr:`required_keys` - """ - name = 'db' - interface = 'mysql' - - def __init__(self, *args, **kwargs): - self.required_keys = ['host', 'user', 'password', 'database'] - RelationContext.__init__(self, *args, **kwargs) - - -class HttpRelation(RelationContext): - """ - Relation context for the `http` interface. - - :param str name: Override the relation :attr:`name`, since it can vary from charm to charm - :param list additional_required_keys: Extend the list of :attr:`required_keys` - """ - name = 'website' - interface = 'http' - - def __init__(self, *args, **kwargs): - self.required_keys = ['host', 'port'] - RelationContext.__init__(self, *args, **kwargs) - - def provide_data(self): - return { - 'host': hookenv.unit_get('private-address'), - 'port': 80, - } - - -class RequiredConfig(dict): - """ - Data context that loads config options with one or more mandatory options. - - Once the required options have been changed from their default values, all - config options will be available, namespaced under `config` to prevent - potential naming conflicts (for example, between a config option and a - relation property). - - :param list *args: List of options that must be changed from their default values. - """ - - def __init__(self, *args): - self.required_options = args - self['config'] = hookenv.config() - with open(os.path.join(hookenv.charm_dir(), 'config.yaml')) as fp: - self.config = yaml.load(fp).get('options', {}) - - def __bool__(self): - for option in self.required_options: - if option not in self['config']: - return False - current_value = self['config'][option] - default_value = self.config[option].get('default') - if current_value == default_value: - return False - if current_value in (None, '') and default_value in (None, ''): - return False - return True - - def __nonzero__(self): - return self.__bool__() - - -class StoredContext(dict): - """ - A data context that always returns the data that it was first created with. - - This is useful to do a one-time generation of things like passwords, that - will thereafter use the same value that was originally generated, instead - of generating a new value each time it is run. - """ - def __init__(self, file_name, config_data): - """ - If the file exists, populate `self` with the data from the file. - Otherwise, populate with the given data and persist it to the file. - """ - if os.path.exists(file_name): - self.update(self.read_context(file_name)) - else: - self.store_context(file_name, config_data) - self.update(config_data) - - def store_context(self, file_name, config_data): - if not os.path.isabs(file_name): - file_name = os.path.join(hookenv.charm_dir(), file_name) - with open(file_name, 'w') as file_stream: - os.fchmod(file_stream.fileno(), 0o600) - yaml.dump(config_data, file_stream) - - def read_context(self, file_name): - if not os.path.isabs(file_name): - file_name = os.path.join(hookenv.charm_dir(), file_name) - with open(file_name, 'r') as file_stream: - data = yaml.load(file_stream) - if not data: - raise OSError("%s is empty" % file_name) - return data - - -class TemplateCallback(ManagerCallback): - """ - Callback class that will render a Jinja2 template, for use as a ready - action. - - :param str source: The template source file, relative to - `$CHARM_DIR/templates` - - :param str target: The target to write the rendered template to (or None) - :param str owner: The owner of the rendered file - :param str group: The group of the rendered file - :param int perms: The permissions of the rendered file - :param partial on_change_action: functools partial to be executed when - rendered file changes - :param jinja2 loader template_loader: A jinja2 template loader - - :return str: The rendered template - """ - def __init__(self, source, target, - owner='root', group='root', perms=0o444, - on_change_action=None, template_loader=None): - self.source = source - self.target = target - self.owner = owner - self.group = group - self.perms = perms - self.on_change_action = on_change_action - self.template_loader = template_loader - - def __call__(self, manager, service_name, event_name): - pre_checksum = '' - if self.on_change_action and os.path.isfile(self.target): - pre_checksum = host.file_hash(self.target) - service = manager.get_service(service_name) - context = {'ctx': {}} - for ctx in service.get('required_data', []): - context.update(ctx) - context['ctx'].update(ctx) - - result = templating.render(self.source, self.target, context, - self.owner, self.group, self.perms, - template_loader=self.template_loader) - if self.on_change_action: - if pre_checksum == host.file_hash(self.target): - hookenv.log( - 'No change detected: {}'.format(self.target), - hookenv.DEBUG) - else: - self.on_change_action() - - return result - - -# Convenience aliases for templates -render_template = template = TemplateCallback diff --git a/hooks/charmhelpers/core/strutils.py b/hooks/charmhelpers/core/strutils.py deleted file mode 100644 index e8df045..0000000 --- a/hooks/charmhelpers/core/strutils.py +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import six -import re - - -def bool_from_string(value): - """Interpret string value as boolean. - - Returns True if value translates to True otherwise False. - """ - if isinstance(value, six.string_types): - value = six.text_type(value) - else: - msg = "Unable to interpret non-string value '%s' as boolean" % (value) - raise ValueError(msg) - - value = value.strip().lower() - - if value in ['y', 'yes', 'true', 't', 'on']: - return True - elif value in ['n', 'no', 'false', 'f', 'off']: - return False - - msg = "Unable to interpret string value '%s' as boolean" % (value) - raise ValueError(msg) - - -def bytes_from_string(value): - """Interpret human readable string value as bytes. - - Returns int - """ - BYTE_POWER = { - 'K': 1, - 'KB': 1, - 'M': 2, - 'MB': 2, - 'G': 3, - 'GB': 3, - 'T': 4, - 'TB': 4, - 'P': 5, - 'PB': 5, - } - if isinstance(value, six.string_types): - value = six.text_type(value) - else: - msg = "Unable to interpret non-string value '%s' as bytes" % (value) - raise ValueError(msg) - matches = re.match("([0-9]+)([a-zA-Z]+)", value) - if matches: - size = int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)]) - else: - # Assume that value passed in is bytes - try: - size = int(value) - except ValueError: - msg = "Unable to interpret string value '%s' as bytes" % (value) - raise ValueError(msg) - return size - - -class BasicStringComparator(object): - """Provides a class that will compare strings from an iterator type object. - Used to provide > and < comparisons on strings that may not necessarily be - alphanumerically ordered. e.g. OpenStack or Ubuntu releases AFTER the - z-wrap. - """ - - _list = None - - def __init__(self, item): - if self._list is None: - raise Exception("Must define the _list in the class definition!") - try: - self.index = self._list.index(item) - except Exception: - raise KeyError("Item '{}' is not in list '{}'" - .format(item, self._list)) - - def __eq__(self, other): - assert isinstance(other, str) or isinstance(other, self.__class__) - return self.index == self._list.index(other) - - def __ne__(self, other): - return not self.__eq__(other) - - def __lt__(self, other): - assert isinstance(other, str) or isinstance(other, self.__class__) - return self.index < self._list.index(other) - - def __ge__(self, other): - return not self.__lt__(other) - - def __gt__(self, other): - assert isinstance(other, str) or isinstance(other, self.__class__) - return self.index > self._list.index(other) - - def __le__(self, other): - return not self.__gt__(other) - - def __str__(self): - """Always give back the item at the index so it can be used in - comparisons like: - - s_mitaka = CompareOpenStack('mitaka') - s_newton = CompareOpenstack('newton') - - assert s_newton > s_mitaka - - @returns: - """ - return self._list[self.index] diff --git a/hooks/charmhelpers/core/sysctl.py b/hooks/charmhelpers/core/sysctl.py deleted file mode 100644 index 6e413e3..0000000 --- a/hooks/charmhelpers/core/sysctl.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import yaml - -from subprocess import check_call - -from charmhelpers.core.hookenv import ( - log, - DEBUG, - ERROR, -) - -__author__ = 'Jorge Niedbalski R. ' - - -def create(sysctl_dict, sysctl_file): - """Creates a sysctl.conf file from a YAML associative array - - :param sysctl_dict: a YAML-formatted string of sysctl options eg "{ 'kernel.max_pid': 1337 }" - :type sysctl_dict: str - :param sysctl_file: path to the sysctl file to be saved - :type sysctl_file: str or unicode - :returns: None - """ - try: - sysctl_dict_parsed = yaml.safe_load(sysctl_dict) - except yaml.YAMLError: - log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict), - level=ERROR) - return - - with open(sysctl_file, "w") as fd: - for key, value in sysctl_dict_parsed.items(): - fd.write("{}={}\n".format(key, value)) - - log("Updating sysctl_file: %s values: %s" % (sysctl_file, sysctl_dict_parsed), - level=DEBUG) - - check_call(["sysctl", "-p", sysctl_file]) diff --git a/hooks/charmhelpers/core/templating.py b/hooks/charmhelpers/core/templating.py deleted file mode 100644 index 7b801a3..0000000 --- a/hooks/charmhelpers/core/templating.py +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys - -from charmhelpers.core import host -from charmhelpers.core import hookenv - - -def render(source, target, context, owner='root', group='root', - perms=0o444, templates_dir=None, encoding='UTF-8', template_loader=None): - """ - Render a template. - - The `source` path, if not absolute, is relative to the `templates_dir`. - - The `target` path should be absolute. It can also be `None`, in which - case no file will be written. - - The context should be a dict containing the values to be replaced in the - template. - - The `owner`, `group`, and `perms` options will be passed to `write_file`. - - If omitted, `templates_dir` defaults to the `templates` folder in the charm. - - The rendered template will be written to the file as well as being returned - as a string. - - Note: Using this requires python-jinja2 or python3-jinja2; if it is not - installed, calling this will attempt to use charmhelpers.fetch.apt_install - to install it. - """ - try: - from jinja2 import FileSystemLoader, Environment, exceptions - except ImportError: - try: - from charmhelpers.fetch import apt_install - except ImportError: - hookenv.log('Could not import jinja2, and could not import ' - 'charmhelpers.fetch to install it', - level=hookenv.ERROR) - raise - if sys.version_info.major == 2: - apt_install('python-jinja2', fatal=True) - else: - apt_install('python3-jinja2', fatal=True) - from jinja2 import FileSystemLoader, Environment, exceptions - - if template_loader: - template_env = Environment(loader=template_loader) - else: - if templates_dir is None: - templates_dir = os.path.join(hookenv.charm_dir(), 'templates') - template_env = Environment(loader=FileSystemLoader(templates_dir)) - try: - source = source - template = template_env.get_template(source) - except exceptions.TemplateNotFound as e: - hookenv.log('Could not load template %s from %s.' % - (source, templates_dir), - level=hookenv.ERROR) - raise e - content = template.render(context) - if target is not None: - target_dir = os.path.dirname(target) - if not os.path.exists(target_dir): - # This is a terrible default directory permission, as the file - # or its siblings will often contain secrets. - host.mkdir(os.path.dirname(target), owner, group, perms=0o755) - host.write_file(target, content.encode(encoding), owner, group, perms) - return content diff --git a/hooks/charmhelpers/core/unitdata.py b/hooks/charmhelpers/core/unitdata.py deleted file mode 100644 index 7af875c..0000000 --- a/hooks/charmhelpers/core/unitdata.py +++ /dev/null @@ -1,518 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Authors: -# Kapil Thangavelu -# -""" -Intro ------ - -A simple way to store state in units. This provides a key value -storage with support for versioned, transactional operation, -and can calculate deltas from previous values to simplify unit logic -when processing changes. - - -Hook Integration ----------------- - -There are several extant frameworks for hook execution, including - - - charmhelpers.core.hookenv.Hooks - - charmhelpers.core.services.ServiceManager - -The storage classes are framework agnostic, one simple integration is -via the HookData contextmanager. It will record the current hook -execution environment (including relation data, config data, etc.), -setup a transaction and allow easy access to the changes from -previously seen values. One consequence of the integration is the -reservation of particular keys ('rels', 'unit', 'env', 'config', -'charm_revisions') for their respective values. - -Here's a fully worked integration example using hookenv.Hooks:: - - from charmhelper.core import hookenv, unitdata - - hook_data = unitdata.HookData() - db = unitdata.kv() - hooks = hookenv.Hooks() - - @hooks.hook - def config_changed(): - # Print all changes to configuration from previously seen - # values. - for changed, (prev, cur) in hook_data.conf.items(): - print('config changed', changed, - 'previous value', prev, - 'current value', cur) - - # Get some unit specific bookeeping - if not db.get('pkg_key'): - key = urllib.urlopen('https://example.com/pkg_key').read() - db.set('pkg_key', key) - - # Directly access all charm config as a mapping. - conf = db.getrange('config', True) - - # Directly access all relation data as a mapping - rels = db.getrange('rels', True) - - if __name__ == '__main__': - with hook_data(): - hook.execute() - - -A more basic integration is via the hook_scope context manager which simply -manages transaction scope (and records hook name, and timestamp):: - - >>> from unitdata import kv - >>> db = kv() - >>> with db.hook_scope('install'): - ... # do work, in transactional scope. - ... db.set('x', 1) - >>> db.get('x') - 1 - - -Usage ------ - -Values are automatically json de/serialized to preserve basic typing -and complex data struct capabilities (dicts, lists, ints, booleans, etc). - -Individual values can be manipulated via get/set:: - - >>> kv.set('y', True) - >>> kv.get('y') - True - - # We can set complex values (dicts, lists) as a single key. - >>> kv.set('config', {'a': 1, 'b': True'}) - - # Also supports returning dictionaries as a record which - # provides attribute access. - >>> config = kv.get('config', record=True) - >>> config.b - True - - -Groups of keys can be manipulated with update/getrange:: - - >>> kv.update({'z': 1, 'y': 2}, prefix="gui.") - >>> kv.getrange('gui.', strip=True) - {'z': 1, 'y': 2} - -When updating values, its very helpful to understand which values -have actually changed and how have they changed. The storage -provides a delta method to provide for this:: - - >>> data = {'debug': True, 'option': 2} - >>> delta = kv.delta(data, 'config.') - >>> delta.debug.previous - None - >>> delta.debug.current - True - >>> delta - {'debug': (None, True), 'option': (None, 2)} - -Note the delta method does not persist the actual change, it needs to -be explicitly saved via 'update' method:: - - >>> kv.update(data, 'config.') - -Values modified in the context of a hook scope retain historical values -associated to the hookname. - - >>> with db.hook_scope('config-changed'): - ... db.set('x', 42) - >>> db.gethistory('x') - [(1, u'x', 1, u'install', u'2015-01-21T16:49:30.038372'), - (2, u'x', 42, u'config-changed', u'2015-01-21T16:49:30.038786')] - -""" - -import collections -import contextlib -import datetime -import itertools -import json -import os -import pprint -import sqlite3 -import sys - -__author__ = 'Kapil Thangavelu ' - - -class Storage(object): - """Simple key value database for local unit state within charms. - - Modifications are not persisted unless :meth:`flush` is called. - - To support dicts, lists, integer, floats, and booleans values - are automatically json encoded/decoded. - """ - def __init__(self, path=None): - self.db_path = path - if path is None: - if 'UNIT_STATE_DB' in os.environ: - self.db_path = os.environ['UNIT_STATE_DB'] - else: - self.db_path = os.path.join( - os.environ.get('CHARM_DIR', ''), '.unit-state.db') - self.conn = sqlite3.connect('%s' % self.db_path) - self.cursor = self.conn.cursor() - self.revision = None - self._closed = False - self._init() - - def close(self): - if self._closed: - return - self.flush(False) - self.cursor.close() - self.conn.close() - self._closed = True - - def get(self, key, default=None, record=False): - self.cursor.execute('select data from kv where key=?', [key]) - result = self.cursor.fetchone() - if not result: - return default - if record: - return Record(json.loads(result[0])) - return json.loads(result[0]) - - def getrange(self, key_prefix, strip=False): - """ - Get a range of keys starting with a common prefix as a mapping of - keys to values. - - :param str key_prefix: Common prefix among all keys - :param bool strip: Optionally strip the common prefix from the key - names in the returned dict - :return dict: A (possibly empty) dict of key-value mappings - """ - self.cursor.execute("select key, data from kv where key like ?", - ['%s%%' % key_prefix]) - result = self.cursor.fetchall() - - if not result: - return {} - if not strip: - key_prefix = '' - return dict([ - (k[len(key_prefix):], json.loads(v)) for k, v in result]) - - def update(self, mapping, prefix=""): - """ - Set the values of multiple keys at once. - - :param dict mapping: Mapping of keys to values - :param str prefix: Optional prefix to apply to all keys in `mapping` - before setting - """ - for k, v in mapping.items(): - self.set("%s%s" % (prefix, k), v) - - def unset(self, key): - """ - Remove a key from the database entirely. - """ - self.cursor.execute('delete from kv where key=?', [key]) - if self.revision and self.cursor.rowcount: - self.cursor.execute( - 'insert into kv_revisions values (?, ?, ?)', - [key, self.revision, json.dumps('DELETED')]) - - def unsetrange(self, keys=None, prefix=""): - """ - Remove a range of keys starting with a common prefix, from the database - entirely. - - :param list keys: List of keys to remove. - :param str prefix: Optional prefix to apply to all keys in ``keys`` - before removing. - """ - if keys is not None: - keys = ['%s%s' % (prefix, key) for key in keys] - self.cursor.execute('delete from kv where key in (%s)' % ','.join(['?'] * len(keys)), keys) - if self.revision and self.cursor.rowcount: - self.cursor.execute( - 'insert into kv_revisions values %s' % ','.join(['(?, ?, ?)'] * len(keys)), - list(itertools.chain.from_iterable((key, self.revision, json.dumps('DELETED')) for key in keys))) - else: - self.cursor.execute('delete from kv where key like ?', - ['%s%%' % prefix]) - if self.revision and self.cursor.rowcount: - self.cursor.execute( - 'insert into kv_revisions values (?, ?, ?)', - ['%s%%' % prefix, self.revision, json.dumps('DELETED')]) - - def set(self, key, value): - """ - Set a value in the database. - - :param str key: Key to set the value for - :param value: Any JSON-serializable value to be set - """ - serialized = json.dumps(value) - - self.cursor.execute('select data from kv where key=?', [key]) - exists = self.cursor.fetchone() - - # Skip mutations to the same value - if exists: - if exists[0] == serialized: - return value - - if not exists: - self.cursor.execute( - 'insert into kv (key, data) values (?, ?)', - (key, serialized)) - else: - self.cursor.execute(''' - update kv - set data = ? - where key = ?''', [serialized, key]) - - # Save - if not self.revision: - return value - - self.cursor.execute( - 'select 1 from kv_revisions where key=? and revision=?', - [key, self.revision]) - exists = self.cursor.fetchone() - - if not exists: - self.cursor.execute( - '''insert into kv_revisions ( - revision, key, data) values (?, ?, ?)''', - (self.revision, key, serialized)) - else: - self.cursor.execute( - ''' - update kv_revisions - set data = ? - where key = ? - and revision = ?''', - [serialized, key, self.revision]) - - return value - - def delta(self, mapping, prefix): - """ - return a delta containing values that have changed. - """ - previous = self.getrange(prefix, strip=True) - if not previous: - pk = set() - else: - pk = set(previous.keys()) - ck = set(mapping.keys()) - delta = DeltaSet() - - # added - for k in ck.difference(pk): - delta[k] = Delta(None, mapping[k]) - - # removed - for k in pk.difference(ck): - delta[k] = Delta(previous[k], None) - - # changed - for k in pk.intersection(ck): - c = mapping[k] - p = previous[k] - if c != p: - delta[k] = Delta(p, c) - - return delta - - @contextlib.contextmanager - def hook_scope(self, name=""): - """Scope all future interactions to the current hook execution - revision.""" - assert not self.revision - self.cursor.execute( - 'insert into hooks (hook, date) values (?, ?)', - (name or sys.argv[0], - datetime.datetime.utcnow().isoformat())) - self.revision = self.cursor.lastrowid - try: - yield self.revision - self.revision = None - except Exception: - self.flush(False) - self.revision = None - raise - else: - self.flush() - - def flush(self, save=True): - if save: - self.conn.commit() - elif self._closed: - return - else: - self.conn.rollback() - - def _init(self): - self.cursor.execute(''' - create table if not exists kv ( - key text, - data text, - primary key (key) - )''') - self.cursor.execute(''' - create table if not exists kv_revisions ( - key text, - revision integer, - data text, - primary key (key, revision) - )''') - self.cursor.execute(''' - create table if not exists hooks ( - version integer primary key autoincrement, - hook text, - date text - )''') - self.conn.commit() - - def gethistory(self, key, deserialize=False): - self.cursor.execute( - ''' - select kv.revision, kv.key, kv.data, h.hook, h.date - from kv_revisions kv, - hooks h - where kv.key=? - and kv.revision = h.version - ''', [key]) - if deserialize is False: - return self.cursor.fetchall() - return map(_parse_history, self.cursor.fetchall()) - - def debug(self, fh=sys.stderr): - self.cursor.execute('select * from kv') - pprint.pprint(self.cursor.fetchall(), stream=fh) - self.cursor.execute('select * from kv_revisions') - pprint.pprint(self.cursor.fetchall(), stream=fh) - - -def _parse_history(d): - return (d[0], d[1], json.loads(d[2]), d[3], - datetime.datetime.strptime(d[-1], "%Y-%m-%dT%H:%M:%S.%f")) - - -class HookData(object): - """Simple integration for existing hook exec frameworks. - - Records all unit information, and stores deltas for processing - by the hook. - - Sample:: - - from charmhelper.core import hookenv, unitdata - - changes = unitdata.HookData() - db = unitdata.kv() - hooks = hookenv.Hooks() - - @hooks.hook - def config_changed(): - # View all changes to configuration - for changed, (prev, cur) in changes.conf.items(): - print('config changed', changed, - 'previous value', prev, - 'current value', cur) - - # Get some unit specific bookeeping - if not db.get('pkg_key'): - key = urllib.urlopen('https://example.com/pkg_key').read() - db.set('pkg_key', key) - - if __name__ == '__main__': - with changes(): - hook.execute() - - """ - def __init__(self): - self.kv = kv() - self.conf = None - self.rels = None - - @contextlib.contextmanager - def __call__(self): - from charmhelpers.core import hookenv - hook_name = hookenv.hook_name() - - with self.kv.hook_scope(hook_name): - self._record_charm_version(hookenv.charm_dir()) - delta_config, delta_relation = self._record_hook(hookenv) - yield self.kv, delta_config, delta_relation - - def _record_charm_version(self, charm_dir): - # Record revisions.. charm revisions are meaningless - # to charm authors as they don't control the revision. - # so logic dependnent on revision is not particularly - # useful, however it is useful for debugging analysis. - charm_rev = open( - os.path.join(charm_dir, 'revision')).read().strip() - charm_rev = charm_rev or '0' - revs = self.kv.get('charm_revisions', []) - if charm_rev not in revs: - revs.append(charm_rev.strip() or '0') - self.kv.set('charm_revisions', revs) - - def _record_hook(self, hookenv): - data = hookenv.execution_environment() - self.conf = conf_delta = self.kv.delta(data['conf'], 'config') - self.rels = rels_delta = self.kv.delta(data['rels'], 'rels') - self.kv.set('env', dict(data['env'])) - self.kv.set('unit', data['unit']) - self.kv.set('relid', data.get('relid')) - return conf_delta, rels_delta - - -class Record(dict): - - __slots__ = () - - def __getattr__(self, k): - if k in self: - return self[k] - raise AttributeError(k) - - -class DeltaSet(Record): - - __slots__ = () - - -Delta = collections.namedtuple('Delta', ['previous', 'current']) - - -_KV = None - - -def kv(): - global _KV - if _KV is None: - _KV = Storage() - return _KV diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py deleted file mode 100644 index 480a627..0000000 --- a/hooks/charmhelpers/fetch/__init__.py +++ /dev/null @@ -1,205 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import importlib -from charmhelpers.osplatform import get_platform -from yaml import safe_load -from charmhelpers.core.hookenv import ( - config, - log, -) - -import six -if six.PY3: - from urllib.parse import urlparse, urlunparse -else: - from urlparse import urlparse, urlunparse - - -# The order of this list is very important. Handlers should be listed in from -# least- to most-specific URL matching. -FETCH_HANDLERS = ( - 'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler', - 'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler', - 'charmhelpers.fetch.giturl.GitUrlFetchHandler', -) - - -class SourceConfigError(Exception): - pass - - -class UnhandledSource(Exception): - pass - - -class AptLockError(Exception): - pass - - -class GPGKeyError(Exception): - """Exception occurs when a GPG key cannot be fetched or used. The message - indicates what the problem is. - """ - pass - - -class BaseFetchHandler(object): - - """Base class for FetchHandler implementations in fetch plugins""" - - def can_handle(self, source): - """Returns True if the source can be handled. Otherwise returns - a string explaining why it cannot""" - return "Wrong source type" - - def install(self, source): - """Try to download and unpack the source. Return the path to the - unpacked files or raise UnhandledSource.""" - raise UnhandledSource("Wrong source type {}".format(source)) - - def parse_url(self, url): - return urlparse(url) - - def base_url(self, url): - """Return url without querystring or fragment""" - parts = list(self.parse_url(url)) - parts[4:] = ['' for i in parts[4:]] - return urlunparse(parts) - - -__platform__ = get_platform() -module = "charmhelpers.fetch.%s" % __platform__ -fetch = importlib.import_module(module) - -filter_installed_packages = fetch.filter_installed_packages -install = fetch.apt_install -upgrade = fetch.apt_upgrade -update = _fetch_update = fetch.apt_update -purge = fetch.apt_purge -add_source = fetch.add_source - -if __platform__ == "ubuntu": - apt_cache = fetch.apt_cache - apt_install = fetch.apt_install - apt_update = fetch.apt_update - apt_upgrade = fetch.apt_upgrade - apt_purge = fetch.apt_purge - apt_mark = fetch.apt_mark - apt_hold = fetch.apt_hold - apt_unhold = fetch.apt_unhold - import_key = fetch.import_key - get_upstream_version = fetch.get_upstream_version -elif __platform__ == "centos": - yum_search = fetch.yum_search - - -def configure_sources(update=False, - sources_var='install_sources', - keys_var='install_keys'): - """Configure multiple sources from charm configuration. - - The lists are encoded as yaml fragments in the configuration. - The fragment needs to be included as a string. Sources and their - corresponding keys are of the types supported by add_source(). - - Example config: - install_sources: | - - "ppa:foo" - - "http://example.com/repo precise main" - install_keys: | - - null - - "a1b2c3d4" - - Note that 'null' (a.k.a. None) should not be quoted. - """ - sources = safe_load((config(sources_var) or '').strip()) or [] - keys = safe_load((config(keys_var) or '').strip()) or None - - if isinstance(sources, six.string_types): - sources = [sources] - - if keys is None: - for source in sources: - add_source(source, None) - else: - if isinstance(keys, six.string_types): - keys = [keys] - - if len(sources) != len(keys): - raise SourceConfigError( - 'Install sources and keys lists are different lengths') - for source, key in zip(sources, keys): - add_source(source, key) - if update: - _fetch_update(fatal=True) - - -def install_remote(source, *args, **kwargs): - """Install a file tree from a remote source. - - The specified source should be a url of the form: - scheme://[host]/path[#[option=value][&...]] - - Schemes supported are based on this modules submodules. - Options supported are submodule-specific. - Additional arguments are passed through to the submodule. - - For example:: - - dest = install_remote('http://example.com/archive.tgz', - checksum='deadbeef', - hash_type='sha1') - - This will download `archive.tgz`, validate it using SHA1 and, if - the file is ok, extract it and return the directory in which it - was extracted. If the checksum fails, it will raise - :class:`charmhelpers.core.host.ChecksumError`. - """ - # We ONLY check for True here because can_handle may return a string - # explaining why it can't handle a given source. - handlers = [h for h in plugins() if h.can_handle(source) is True] - for handler in handlers: - try: - return handler.install(source, *args, **kwargs) - except UnhandledSource as e: - log('Install source attempt unsuccessful: {}'.format(e), - level='WARNING') - raise UnhandledSource("No handler found for source {}".format(source)) - - -def install_from_config(config_var_name): - """Install a file from config.""" - charm_config = config() - source = charm_config[config_var_name] - return install_remote(source) - - -def plugins(fetch_handlers=None): - if not fetch_handlers: - fetch_handlers = FETCH_HANDLERS - plugin_list = [] - for handler_name in fetch_handlers: - package, classname = handler_name.rsplit('.', 1) - try: - handler_class = getattr( - importlib.import_module(package), - classname) - plugin_list.append(handler_class()) - except NotImplementedError: - # Skip missing plugins so that they can be ommitted from - # installation if desired - log("FetchHandler {} not found, skipping plugin".format( - handler_name)) - return plugin_list diff --git a/hooks/charmhelpers/fetch/archiveurl.py b/hooks/charmhelpers/fetch/archiveurl.py deleted file mode 100644 index dd24f9e..0000000 --- a/hooks/charmhelpers/fetch/archiveurl.py +++ /dev/null @@ -1,165 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import hashlib -import re - -from charmhelpers.fetch import ( - BaseFetchHandler, - UnhandledSource -) -from charmhelpers.payload.archive import ( - get_archive_handler, - extract, -) -from charmhelpers.core.host import mkdir, check_hash - -import six -if six.PY3: - from urllib.request import ( - build_opener, install_opener, urlopen, urlretrieve, - HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, - ) - from urllib.parse import urlparse, urlunparse, parse_qs - from urllib.error import URLError -else: - from urllib import urlretrieve - from urllib2 import ( - build_opener, install_opener, urlopen, - HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, - URLError - ) - from urlparse import urlparse, urlunparse, parse_qs - - -def splituser(host): - '''urllib.splituser(), but six's support of this seems broken''' - _userprog = re.compile('^(.*)@(.*)$') - match = _userprog.match(host) - if match: - return match.group(1, 2) - return None, host - - -def splitpasswd(user): - '''urllib.splitpasswd(), but six's support of this is missing''' - _passwdprog = re.compile('^([^:]*):(.*)$', re.S) - match = _passwdprog.match(user) - if match: - return match.group(1, 2) - return user, None - - -class ArchiveUrlFetchHandler(BaseFetchHandler): - """ - Handler to download archive files from arbitrary URLs. - - Can fetch from http, https, ftp, and file URLs. - - Can install either tarballs (.tar, .tgz, .tbz2, etc) or zip files. - - Installs the contents of the archive in $CHARM_DIR/fetched/. - """ - def can_handle(self, source): - url_parts = self.parse_url(source) - if url_parts.scheme not in ('http', 'https', 'ftp', 'file'): - # XXX: Why is this returning a boolean and a string? It's - # doomed to fail since "bool(can_handle('foo://'))" will be True. - return "Wrong source type" - if get_archive_handler(self.base_url(source)): - return True - return False - - def download(self, source, dest): - """ - Download an archive file. - - :param str source: URL pointing to an archive file. - :param str dest: Local path location to download archive file to. - """ - # propogate all exceptions - # URLError, OSError, etc - proto, netloc, path, params, query, fragment = urlparse(source) - if proto in ('http', 'https'): - auth, barehost = splituser(netloc) - if auth is not None: - source = urlunparse((proto, barehost, path, params, query, fragment)) - username, password = splitpasswd(auth) - passman = HTTPPasswordMgrWithDefaultRealm() - # Realm is set to None in add_password to force the username and password - # to be used whatever the realm - passman.add_password(None, source, username, password) - authhandler = HTTPBasicAuthHandler(passman) - opener = build_opener(authhandler) - install_opener(opener) - response = urlopen(source) - try: - with open(dest, 'wb') as dest_file: - dest_file.write(response.read()) - except Exception as e: - if os.path.isfile(dest): - os.unlink(dest) - raise e - - # Mandatory file validation via Sha1 or MD5 hashing. - def download_and_validate(self, url, hashsum, validate="sha1"): - tempfile, headers = urlretrieve(url) - check_hash(tempfile, hashsum, validate) - return tempfile - - def install(self, source, dest=None, checksum=None, hash_type='sha1'): - """ - Download and install an archive file, with optional checksum validation. - - The checksum can also be given on the `source` URL's fragment. - For example:: - - handler.install('http://example.com/file.tgz#sha1=deadbeef') - - :param str source: URL pointing to an archive file. - :param str dest: Local destination path to install to. If not given, - installs to `$CHARM_DIR/archives/archive_file_name`. - :param str checksum: If given, validate the archive file after download. - :param str hash_type: Algorithm used to generate `checksum`. - Can be any hash alrgorithm supported by :mod:`hashlib`, - such as md5, sha1, sha256, sha512, etc. - - """ - url_parts = self.parse_url(source) - dest_dir = os.path.join(os.environ.get('CHARM_DIR'), 'fetched') - if not os.path.exists(dest_dir): - mkdir(dest_dir, perms=0o755) - dld_file = os.path.join(dest_dir, os.path.basename(url_parts.path)) - try: - self.download(source, dld_file) - except URLError as e: - raise UnhandledSource(e.reason) - except OSError as e: - raise UnhandledSource(e.strerror) - options = parse_qs(url_parts.fragment) - for key, value in options.items(): - if not six.PY3: - algorithms = hashlib.algorithms - else: - algorithms = hashlib.algorithms_available - if key in algorithms: - if len(value) != 1: - raise TypeError( - "Expected 1 hash value, not %d" % len(value)) - expected = value[0] - check_hash(dld_file, expected, key) - if checksum: - check_hash(dld_file, checksum, hash_type) - return extract(dld_file, dest) diff --git a/hooks/charmhelpers/fetch/bzrurl.py b/hooks/charmhelpers/fetch/bzrurl.py deleted file mode 100644 index 07cd029..0000000 --- a/hooks/charmhelpers/fetch/bzrurl.py +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -from subprocess import check_call -from charmhelpers.fetch import ( - BaseFetchHandler, - UnhandledSource, - filter_installed_packages, - install, -) -from charmhelpers.core.host import mkdir - - -if filter_installed_packages(['bzr']) != []: - install(['bzr']) - if filter_installed_packages(['bzr']) != []: - raise NotImplementedError('Unable to install bzr') - - -class BzrUrlFetchHandler(BaseFetchHandler): - """Handler for bazaar branches via generic and lp URLs.""" - - def can_handle(self, source): - url_parts = self.parse_url(source) - if url_parts.scheme not in ('bzr+ssh', 'lp', ''): - return False - elif not url_parts.scheme: - return os.path.exists(os.path.join(source, '.bzr')) - else: - return True - - def branch(self, source, dest, revno=None): - if not self.can_handle(source): - raise UnhandledSource("Cannot handle {}".format(source)) - cmd_opts = [] - if revno: - cmd_opts += ['-r', str(revno)] - if os.path.exists(dest): - cmd = ['bzr', 'pull'] - cmd += cmd_opts - cmd += ['--overwrite', '-d', dest, source] - else: - cmd = ['bzr', 'branch'] - cmd += cmd_opts - cmd += [source, dest] - check_call(cmd) - - def install(self, source, dest=None, revno=None): - url_parts = self.parse_url(source) - branch_name = url_parts.path.strip("/").split("/")[-1] - if dest: - dest_dir = os.path.join(dest, branch_name) - else: - dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", - branch_name) - - if dest and not os.path.exists(dest): - mkdir(dest, perms=0o755) - - try: - self.branch(source, dest_dir, revno) - except OSError as e: - raise UnhandledSource(e.strerror) - return dest_dir diff --git a/hooks/charmhelpers/fetch/centos.py b/hooks/charmhelpers/fetch/centos.py deleted file mode 100644 index a91dcff..0000000 --- a/hooks/charmhelpers/fetch/centos.py +++ /dev/null @@ -1,171 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import subprocess -import os -import time -import six -import yum - -from tempfile import NamedTemporaryFile -from charmhelpers.core.hookenv import log - -YUM_NO_LOCK = 1 # The return code for "couldn't acquire lock" in YUM. -YUM_NO_LOCK_RETRY_DELAY = 10 # Wait 10 seconds between apt lock checks. -YUM_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times. - - -def filter_installed_packages(packages): - """Return a list of packages that require installation.""" - yb = yum.YumBase() - package_list = yb.doPackageLists() - temp_cache = {p.base_package_name: 1 for p in package_list['installed']} - - _pkgs = [p for p in packages if not temp_cache.get(p, False)] - return _pkgs - - -def install(packages, options=None, fatal=False): - """Install one or more packages.""" - cmd = ['yum', '--assumeyes'] - if options is not None: - cmd.extend(options) - cmd.append('install') - if isinstance(packages, six.string_types): - cmd.append(packages) - else: - cmd.extend(packages) - log("Installing {} with options: {}".format(packages, - options)) - _run_yum_command(cmd, fatal) - - -def upgrade(options=None, fatal=False, dist=False): - """Upgrade all packages.""" - cmd = ['yum', '--assumeyes'] - if options is not None: - cmd.extend(options) - cmd.append('upgrade') - log("Upgrading with options: {}".format(options)) - _run_yum_command(cmd, fatal) - - -def update(fatal=False): - """Update local yum cache.""" - cmd = ['yum', '--assumeyes', 'update'] - log("Update with fatal: {}".format(fatal)) - _run_yum_command(cmd, fatal) - - -def purge(packages, fatal=False): - """Purge one or more packages.""" - cmd = ['yum', '--assumeyes', 'remove'] - if isinstance(packages, six.string_types): - cmd.append(packages) - else: - cmd.extend(packages) - log("Purging {}".format(packages)) - _run_yum_command(cmd, fatal) - - -def yum_search(packages): - """Search for a package.""" - output = {} - cmd = ['yum', 'search'] - if isinstance(packages, six.string_types): - cmd.append(packages) - else: - cmd.extend(packages) - log("Searching for {}".format(packages)) - result = subprocess.check_output(cmd) - for package in list(packages): - output[package] = package in result - return output - - -def add_source(source, key=None): - """Add a package source to this system. - - @param source: a URL with a rpm package - - @param key: A key to be added to the system's keyring and used - to verify the signatures on packages. Ideally, this should be an - ASCII format GPG public key including the block headers. A GPG key - id may also be used, but be aware that only insecure protocols are - available to retrieve the actual public key from a public keyserver - placing your Juju environment at risk. - """ - if source is None: - log('Source is not present. Skipping') - return - - if source.startswith('http'): - directory = '/etc/yum.repos.d/' - for filename in os.listdir(directory): - with open(directory + filename, 'r') as rpm_file: - if source in rpm_file.read(): - break - else: - log("Add source: {!r}".format(source)) - # write in the charms.repo - with open(directory + 'Charms.repo', 'a') as rpm_file: - rpm_file.write('[%s]\n' % source[7:].replace('/', '_')) - rpm_file.write('name=%s\n' % source[7:]) - rpm_file.write('baseurl=%s\n\n' % source) - else: - log("Unknown source: {!r}".format(source)) - - if key: - if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key: - with NamedTemporaryFile('w+') as key_file: - key_file.write(key) - key_file.flush() - key_file.seek(0) - subprocess.check_call(['rpm', '--import', key_file.name]) - else: - subprocess.check_call(['rpm', '--import', key]) - - -def _run_yum_command(cmd, fatal=False): - """Run an YUM command. - - Checks the output and retry if the fatal flag is set to True. - - :param: cmd: str: The yum command to run. - :param: fatal: bool: Whether the command's output should be checked and - retried. - """ - env = os.environ.copy() - - if fatal: - retry_count = 0 - result = None - - # If the command is considered "fatal", we need to retry if the yum - # lock was not acquired. - - while result is None or result == YUM_NO_LOCK: - try: - result = subprocess.check_call(cmd, env=env) - except subprocess.CalledProcessError as e: - retry_count = retry_count + 1 - if retry_count > YUM_NO_LOCK_RETRY_COUNT: - raise - result = e.returncode - log("Couldn't acquire YUM lock. Will retry in {} seconds." - "".format(YUM_NO_LOCK_RETRY_DELAY)) - time.sleep(YUM_NO_LOCK_RETRY_DELAY) - - else: - subprocess.call(cmd, env=env) diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py deleted file mode 100644 index 4cf21bc..0000000 --- a/hooks/charmhelpers/fetch/giturl.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -from subprocess import check_call, CalledProcessError -from charmhelpers.fetch import ( - BaseFetchHandler, - UnhandledSource, - filter_installed_packages, - install, -) - -if filter_installed_packages(['git']) != []: - install(['git']) - if filter_installed_packages(['git']) != []: - raise NotImplementedError('Unable to install git') - - -class GitUrlFetchHandler(BaseFetchHandler): - """Handler for git branches via generic and github URLs.""" - - def can_handle(self, source): - url_parts = self.parse_url(source) - # TODO (mattyw) no support for ssh git@ yet - if url_parts.scheme not in ('http', 'https', 'git', ''): - return False - elif not url_parts.scheme: - return os.path.exists(os.path.join(source, '.git')) - else: - return True - - def clone(self, source, dest, branch="master", depth=None): - if not self.can_handle(source): - raise UnhandledSource("Cannot handle {}".format(source)) - - if os.path.exists(dest): - cmd = ['git', '-C', dest, 'pull', source, branch] - else: - cmd = ['git', 'clone', source, dest, '--branch', branch] - if depth: - cmd.extend(['--depth', depth]) - check_call(cmd) - - def install(self, source, branch="master", dest=None, depth=None): - url_parts = self.parse_url(source) - branch_name = url_parts.path.strip("/").split("/")[-1] - if dest: - dest_dir = os.path.join(dest, branch_name) - else: - dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", - branch_name) - try: - self.clone(source, dest_dir, branch, depth) - except CalledProcessError as e: - raise UnhandledSource(e) - except OSError as e: - raise UnhandledSource(e.strerror) - return dest_dir diff --git a/hooks/charmhelpers/fetch/snap.py b/hooks/charmhelpers/fetch/snap.py deleted file mode 100644 index 395836c..0000000 --- a/hooks/charmhelpers/fetch/snap.py +++ /dev/null @@ -1,150 +0,0 @@ -# Copyright 2014-2017 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Charm helpers snap for classic charms. - -If writing reactive charms, use the snap layer: -https://lists.ubuntu.com/archives/snapcraft/2016-September/001114.html -""" -import subprocess -import os -from time import sleep -from charmhelpers.core.hookenv import log - -__author__ = 'Joseph Borg ' - -# The return code for "couldn't acquire lock" in Snap -# (hopefully this will be improved). -SNAP_NO_LOCK = 1 -SNAP_NO_LOCK_RETRY_DELAY = 10 # Wait X seconds between Snap lock checks. -SNAP_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times. -SNAP_CHANNELS = [ - 'edge', - 'beta', - 'candidate', - 'stable', -] - - -class CouldNotAcquireLockException(Exception): - pass - - -class InvalidSnapChannel(Exception): - pass - - -def _snap_exec(commands): - """ - Execute snap commands. - - :param commands: List commands - :return: Integer exit code - """ - assert type(commands) == list - - retry_count = 0 - return_code = None - - while return_code is None or return_code == SNAP_NO_LOCK: - try: - return_code = subprocess.check_call(['snap'] + commands, - env=os.environ) - except subprocess.CalledProcessError as e: - retry_count += + 1 - if retry_count > SNAP_NO_LOCK_RETRY_COUNT: - raise CouldNotAcquireLockException( - 'Could not aquire lock after {} attempts' - .format(SNAP_NO_LOCK_RETRY_COUNT)) - return_code = e.returncode - log('Snap failed to acquire lock, trying again in {} seconds.' - .format(SNAP_NO_LOCK_RETRY_DELAY, level='WARN')) - sleep(SNAP_NO_LOCK_RETRY_DELAY) - - return return_code - - -def snap_install(packages, *flags): - """ - Install a snap package. - - :param packages: String or List String package name - :param flags: List String flags to pass to install command - :return: Integer return code from snap - """ - if type(packages) is not list: - packages = [packages] - - flags = list(flags) - - message = 'Installing snap(s) "%s"' % ', '.join(packages) - if flags: - message += ' with option(s) "%s"' % ', '.join(flags) - - log(message, level='INFO') - return _snap_exec(['install'] + flags + packages) - - -def snap_remove(packages, *flags): - """ - Remove a snap package. - - :param packages: String or List String package name - :param flags: List String flags to pass to remove command - :return: Integer return code from snap - """ - if type(packages) is not list: - packages = [packages] - - flags = list(flags) - - message = 'Removing snap(s) "%s"' % ', '.join(packages) - if flags: - message += ' with options "%s"' % ', '.join(flags) - - log(message, level='INFO') - return _snap_exec(['remove'] + flags + packages) - - -def snap_refresh(packages, *flags): - """ - Refresh / Update snap package. - - :param packages: String or List String package name - :param flags: List String flags to pass to refresh command - :return: Integer return code from snap - """ - if type(packages) is not list: - packages = [packages] - - flags = list(flags) - - message = 'Refreshing snap(s) "%s"' % ', '.join(packages) - if flags: - message += ' with options "%s"' % ', '.join(flags) - - log(message, level='INFO') - return _snap_exec(['refresh'] + flags + packages) - - -def valid_snap_channel(channel): - """ Validate snap channel exists - - :raises InvalidSnapChannel: When channel does not exist - :return: Boolean - """ - if channel.lower() in SNAP_CHANNELS: - return True - else: - raise InvalidSnapChannel("Invalid Snap Channel: {}".format(channel)) diff --git a/hooks/charmhelpers/fetch/ubuntu.py b/hooks/charmhelpers/fetch/ubuntu.py deleted file mode 100644 index 910e96a..0000000 --- a/hooks/charmhelpers/fetch/ubuntu.py +++ /dev/null @@ -1,583 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from collections import OrderedDict -import os -import platform -import re -import six -import time -import subprocess -from tempfile import NamedTemporaryFile - -from charmhelpers.core.host import ( - lsb_release -) -from charmhelpers.core.hookenv import ( - log, - DEBUG, - WARNING, -) -from charmhelpers.fetch import SourceConfigError, GPGKeyError - -PROPOSED_POCKET = ( - "# Proposed\n" - "deb http://archive.ubuntu.com/ubuntu {}-proposed main universe " - "multiverse restricted\n") -PROPOSED_PORTS_POCKET = ( - "# Proposed\n" - "deb http://ports.ubuntu.com/ubuntu-ports {}-proposed main universe " - "multiverse restricted\n") -# Only supports 64bit and ppc64 at the moment. -ARCH_TO_PROPOSED_POCKET = { - 'x86_64': PROPOSED_POCKET, - 'ppc64le': PROPOSED_PORTS_POCKET, - 'aarch64': PROPOSED_PORTS_POCKET, -} -CLOUD_ARCHIVE_URL = "http://ubuntu-cloud.archive.canonical.com/ubuntu" -CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA' -CLOUD_ARCHIVE = """# Ubuntu Cloud Archive -deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main -""" -CLOUD_ARCHIVE_POCKETS = { - # Folsom - 'folsom': 'precise-updates/folsom', - 'folsom/updates': 'precise-updates/folsom', - 'precise-folsom': 'precise-updates/folsom', - 'precise-folsom/updates': 'precise-updates/folsom', - 'precise-updates/folsom': 'precise-updates/folsom', - 'folsom/proposed': 'precise-proposed/folsom', - 'precise-folsom/proposed': 'precise-proposed/folsom', - 'precise-proposed/folsom': 'precise-proposed/folsom', - # Grizzly - 'grizzly': 'precise-updates/grizzly', - 'grizzly/updates': 'precise-updates/grizzly', - 'precise-grizzly': 'precise-updates/grizzly', - 'precise-grizzly/updates': 'precise-updates/grizzly', - 'precise-updates/grizzly': 'precise-updates/grizzly', - 'grizzly/proposed': 'precise-proposed/grizzly', - 'precise-grizzly/proposed': 'precise-proposed/grizzly', - 'precise-proposed/grizzly': 'precise-proposed/grizzly', - # Havana - 'havana': 'precise-updates/havana', - 'havana/updates': 'precise-updates/havana', - 'precise-havana': 'precise-updates/havana', - 'precise-havana/updates': 'precise-updates/havana', - 'precise-updates/havana': 'precise-updates/havana', - 'havana/proposed': 'precise-proposed/havana', - 'precise-havana/proposed': 'precise-proposed/havana', - 'precise-proposed/havana': 'precise-proposed/havana', - # Icehouse - 'icehouse': 'precise-updates/icehouse', - 'icehouse/updates': 'precise-updates/icehouse', - 'precise-icehouse': 'precise-updates/icehouse', - 'precise-icehouse/updates': 'precise-updates/icehouse', - 'precise-updates/icehouse': 'precise-updates/icehouse', - 'icehouse/proposed': 'precise-proposed/icehouse', - 'precise-icehouse/proposed': 'precise-proposed/icehouse', - 'precise-proposed/icehouse': 'precise-proposed/icehouse', - # Juno - 'juno': 'trusty-updates/juno', - 'juno/updates': 'trusty-updates/juno', - 'trusty-juno': 'trusty-updates/juno', - 'trusty-juno/updates': 'trusty-updates/juno', - 'trusty-updates/juno': 'trusty-updates/juno', - 'juno/proposed': 'trusty-proposed/juno', - 'trusty-juno/proposed': 'trusty-proposed/juno', - 'trusty-proposed/juno': 'trusty-proposed/juno', - # Kilo - 'kilo': 'trusty-updates/kilo', - 'kilo/updates': 'trusty-updates/kilo', - 'trusty-kilo': 'trusty-updates/kilo', - 'trusty-kilo/updates': 'trusty-updates/kilo', - 'trusty-updates/kilo': 'trusty-updates/kilo', - 'kilo/proposed': 'trusty-proposed/kilo', - 'trusty-kilo/proposed': 'trusty-proposed/kilo', - 'trusty-proposed/kilo': 'trusty-proposed/kilo', - # Liberty - 'liberty': 'trusty-updates/liberty', - 'liberty/updates': 'trusty-updates/liberty', - 'trusty-liberty': 'trusty-updates/liberty', - 'trusty-liberty/updates': 'trusty-updates/liberty', - 'trusty-updates/liberty': 'trusty-updates/liberty', - 'liberty/proposed': 'trusty-proposed/liberty', - 'trusty-liberty/proposed': 'trusty-proposed/liberty', - 'trusty-proposed/liberty': 'trusty-proposed/liberty', - # Mitaka - 'mitaka': 'trusty-updates/mitaka', - 'mitaka/updates': 'trusty-updates/mitaka', - 'trusty-mitaka': 'trusty-updates/mitaka', - 'trusty-mitaka/updates': 'trusty-updates/mitaka', - 'trusty-updates/mitaka': 'trusty-updates/mitaka', - 'mitaka/proposed': 'trusty-proposed/mitaka', - 'trusty-mitaka/proposed': 'trusty-proposed/mitaka', - 'trusty-proposed/mitaka': 'trusty-proposed/mitaka', - # Newton - 'newton': 'xenial-updates/newton', - 'newton/updates': 'xenial-updates/newton', - 'xenial-newton': 'xenial-updates/newton', - 'xenial-newton/updates': 'xenial-updates/newton', - 'xenial-updates/newton': 'xenial-updates/newton', - 'newton/proposed': 'xenial-proposed/newton', - 'xenial-newton/proposed': 'xenial-proposed/newton', - 'xenial-proposed/newton': 'xenial-proposed/newton', - # Ocata - 'ocata': 'xenial-updates/ocata', - 'ocata/updates': 'xenial-updates/ocata', - 'xenial-ocata': 'xenial-updates/ocata', - 'xenial-ocata/updates': 'xenial-updates/ocata', - 'xenial-updates/ocata': 'xenial-updates/ocata', - 'ocata/proposed': 'xenial-proposed/ocata', - 'xenial-ocata/proposed': 'xenial-proposed/ocata', - 'xenial-proposed/ocata': 'xenial-proposed/ocata', - # Pike - 'pike': 'xenial-updates/pike', - 'xenial-pike': 'xenial-updates/pike', - 'xenial-pike/updates': 'xenial-updates/pike', - 'xenial-updates/pike': 'xenial-updates/pike', - 'pike/proposed': 'xenial-proposed/pike', - 'xenial-pike/proposed': 'xenial-proposed/pike', - 'xenial-proposed/pike': 'xenial-proposed/pike', - # Queens - 'queens': 'xenial-updates/queens', - 'xenial-queens': 'xenial-updates/queens', - 'xenial-queens/updates': 'xenial-updates/queens', - 'xenial-updates/queens': 'xenial-updates/queens', - 'queens/proposed': 'xenial-proposed/queens', - 'xenial-queens/proposed': 'xenial-proposed/queens', - 'xenial-proposed/queens': 'xenial-proposed/queens', -} - - -APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT. -CMD_RETRY_DELAY = 10 # Wait 10 seconds between command retries. -CMD_RETRY_COUNT = 3 # Retry a failing fatal command X times. - - -def filter_installed_packages(packages): - """Return a list of packages that require installation.""" - cache = apt_cache() - _pkgs = [] - for package in packages: - try: - p = cache[package] - p.current_ver or _pkgs.append(package) - except KeyError: - log('Package {} has no installation candidate.'.format(package), - level='WARNING') - _pkgs.append(package) - return _pkgs - - -def apt_cache(in_memory=True, progress=None): - """Build and return an apt cache.""" - from apt import apt_pkg - apt_pkg.init() - if in_memory: - apt_pkg.config.set("Dir::Cache::pkgcache", "") - apt_pkg.config.set("Dir::Cache::srcpkgcache", "") - return apt_pkg.Cache(progress) - - -def apt_install(packages, options=None, fatal=False): - """Install one or more packages.""" - if options is None: - options = ['--option=Dpkg::Options::=--force-confold'] - - cmd = ['apt-get', '--assume-yes'] - cmd.extend(options) - cmd.append('install') - if isinstance(packages, six.string_types): - cmd.append(packages) - else: - cmd.extend(packages) - log("Installing {} with options: {}".format(packages, - options)) - _run_apt_command(cmd, fatal) - - -def apt_upgrade(options=None, fatal=False, dist=False): - """Upgrade all packages.""" - if options is None: - options = ['--option=Dpkg::Options::=--force-confold'] - - cmd = ['apt-get', '--assume-yes'] - cmd.extend(options) - if dist: - cmd.append('dist-upgrade') - else: - cmd.append('upgrade') - log("Upgrading with options: {}".format(options)) - _run_apt_command(cmd, fatal) - - -def apt_update(fatal=False): - """Update local apt cache.""" - cmd = ['apt-get', 'update'] - _run_apt_command(cmd, fatal) - - -def apt_purge(packages, fatal=False): - """Purge one or more packages.""" - cmd = ['apt-get', '--assume-yes', 'purge'] - if isinstance(packages, six.string_types): - cmd.append(packages) - else: - cmd.extend(packages) - log("Purging {}".format(packages)) - _run_apt_command(cmd, fatal) - - -def apt_mark(packages, mark, fatal=False): - """Flag one or more packages using apt-mark.""" - log("Marking {} as {}".format(packages, mark)) - cmd = ['apt-mark', mark] - if isinstance(packages, six.string_types): - cmd.append(packages) - else: - cmd.extend(packages) - - if fatal: - subprocess.check_call(cmd, universal_newlines=True) - else: - subprocess.call(cmd, universal_newlines=True) - - -def apt_hold(packages, fatal=False): - return apt_mark(packages, 'hold', fatal=fatal) - - -def apt_unhold(packages, fatal=False): - return apt_mark(packages, 'unhold', fatal=fatal) - - -def import_key(key): - """Import an ASCII Armor key. - - /!\ A Radix64 format keyid is also supported for backwards - compatibility, but should never be used; the key retrieval - mechanism is insecure and subject to man-in-the-middle attacks - voiding all signature checks using that key. - - :param keyid: The key in ASCII armor format, - including BEGIN and END markers. - :raises: GPGKeyError if the key could not be imported - """ - key = key.strip() - if '-' in key or '\n' in key: - # Send everything not obviously a keyid to GPG to import, as - # we trust its validation better than our own. eg. handling - # comments before the key. - log("PGP key found (looks like ASCII Armor format)", level=DEBUG) - if ('-----BEGIN PGP PUBLIC KEY BLOCK-----' in key and - '-----END PGP PUBLIC KEY BLOCK-----' in key): - log("Importing ASCII Armor PGP key", level=DEBUG) - with NamedTemporaryFile() as keyfile: - with open(keyfile.name, 'w') as fd: - fd.write(key) - fd.write("\n") - cmd = ['apt-key', 'add', keyfile.name] - try: - subprocess.check_call(cmd) - except subprocess.CalledProcessError: - error = "Error importing PGP key '{}'".format(key) - log(error) - raise GPGKeyError(error) - else: - raise GPGKeyError("ASCII armor markers missing from GPG key") - else: - # We should only send things obviously not a keyid offsite - # via this unsecured protocol, as it may be a secret or part - # of one. - log("PGP key found (looks like Radix64 format)", level=WARNING) - log("INSECURLY importing PGP key from keyserver; " - "full key not provided.", level=WARNING) - cmd = ['apt-key', 'adv', '--keyserver', - 'hkp://keyserver.ubuntu.com:80', '--recv-keys', key] - try: - subprocess.check_call(cmd) - except subprocess.CalledProcessError: - error = "Error importing PGP key '{}'".format(key) - log(error) - raise GPGKeyError(error) - - -def add_source(source, key=None, fail_invalid=False): - """Add a package source to this system. - - @param source: a URL or sources.list entry, as supported by - add-apt-repository(1). Examples:: - - ppa:charmers/example - deb https://stub:key@private.example.com/ubuntu trusty main - - In addition: - 'proposed:' may be used to enable the standard 'proposed' - pocket for the release. - 'cloud:' may be used to activate official cloud archive pockets, - such as 'cloud:icehouse' - 'distro' may be used as a noop - - Full list of source specifications supported by the function are: - - 'distro': A NOP; i.e. it has no effect. - 'proposed': the proposed deb spec [2] is wrtten to - /etc/apt/sources.list/proposed - 'distro-proposed': adds -proposed to the debs [2] - 'ppa:': add-apt-repository --yes - 'deb ': add-apt-repository --yes deb - 'http://....': add-apt-repository --yes http://... - 'cloud-archive:': add-apt-repository -yes cloud-archive: - 'cloud:[-staging]': specify a Cloud Archive pocket with - optional staging version. If staging is used then the staging PPA [2] - with be used. If staging is NOT used then the cloud archive [3] will be - added, and the 'ubuntu-cloud-keyring' package will be added for the - current distro. - - Otherwise the source is not recognised and this is logged to the juju log. - However, no error is raised, unless sys_error_on_exit is True. - - [1] deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main - where {} is replaced with the derived pocket name. - [2] deb http://archive.ubuntu.com/ubuntu {}-proposed \ - main universe multiverse restricted - where {} is replaced with the lsb_release codename (e.g. xenial) - [3] deb http://ubuntu-cloud.archive.canonical.com/ubuntu - to /etc/apt/sources.list.d/cloud-archive-list - - @param key: A key to be added to the system's APT keyring and used - to verify the signatures on packages. Ideally, this should be an - ASCII format GPG public key including the block headers. A GPG key - id may also be used, but be aware that only insecure protocols are - available to retrieve the actual public key from a public keyserver - placing your Juju environment at risk. ppa and cloud archive keys - are securely added automtically, so sould not be provided. - - @param fail_invalid: (boolean) if True, then the function raises a - SourceConfigError is there is no matching installation source. - - @raises SourceConfigError() if for cloud:, the is not a - valid pocket in CLOUD_ARCHIVE_POCKETS - """ - _mapping = OrderedDict([ - (r"^distro$", lambda: None), # This is a NOP - (r"^(?:proposed|distro-proposed)$", _add_proposed), - (r"^cloud-archive:(.*)$", _add_apt_repository), - (r"^((?:deb |http:|https:|ppa:).*)$", _add_apt_repository), - (r"^cloud:(.*)-(.*)\/staging$", _add_cloud_staging), - (r"^cloud:(.*)-(.*)$", _add_cloud_distro_check), - (r"^cloud:(.*)$", _add_cloud_pocket), - (r"^snap:.*-(.*)-(.*)$", _add_cloud_distro_check), - ]) - if source is None: - source = '' - for r, fn in six.iteritems(_mapping): - m = re.match(r, source) - if m: - # call the assoicated function with the captured groups - # raises SourceConfigError on error. - fn(*m.groups()) - if key: - try: - import_key(key) - except GPGKeyError as e: - raise SourceConfigError(str(e)) - break - else: - # nothing matched. log an error and maybe sys.exit - err = "Unknown source: {!r}".format(source) - log(err) - if fail_invalid: - raise SourceConfigError(err) - - -def _add_proposed(): - """Add the PROPOSED_POCKET as /etc/apt/source.list.d/proposed.list - - Uses lsb_release()['DISTRIB_CODENAME'] to determine the correct staza for - the deb line. - - For intel architecutres PROPOSED_POCKET is used for the release, but for - other architectures PROPOSED_PORTS_POCKET is used for the release. - """ - release = lsb_release()['DISTRIB_CODENAME'] - arch = platform.machine() - if arch not in six.iterkeys(ARCH_TO_PROPOSED_POCKET): - raise SourceConfigError("Arch {} not supported for (distro-)proposed" - .format(arch)) - with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt: - apt.write(ARCH_TO_PROPOSED_POCKET[arch].format(release)) - - -def _add_apt_repository(spec): - """Add the spec using add_apt_repository - - :param spec: the parameter to pass to add_apt_repository - """ - _run_with_retries(['add-apt-repository', '--yes', spec]) - - -def _add_cloud_pocket(pocket): - """Add a cloud pocket as /etc/apt/sources.d/cloud-archive.list - - Note that this overwrites the existing file if there is one. - - This function also converts the simple pocket in to the actual pocket using - the CLOUD_ARCHIVE_POCKETS mapping. - - :param pocket: string representing the pocket to add a deb spec for. - :raises: SourceConfigError if the cloud pocket doesn't exist or the - requested release doesn't match the current distro version. - """ - apt_install(filter_installed_packages(['ubuntu-cloud-keyring']), - fatal=True) - if pocket not in CLOUD_ARCHIVE_POCKETS: - raise SourceConfigError( - 'Unsupported cloud: source option %s' % - pocket) - actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket] - with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt: - apt.write(CLOUD_ARCHIVE.format(actual_pocket)) - - -def _add_cloud_staging(cloud_archive_release, openstack_release): - """Add the cloud staging repository which is in - ppa:ubuntu-cloud-archive/-staging - - This function checks that the cloud_archive_release matches the current - codename for the distro that charm is being installed on. - - :param cloud_archive_release: string, codename for the release. - :param openstack_release: String, codename for the openstack release. - :raises: SourceConfigError if the cloud_archive_release doesn't match the - current version of the os. - """ - _verify_is_ubuntu_rel(cloud_archive_release, openstack_release) - ppa = 'ppa:ubuntu-cloud-archive/{}-staging'.format(openstack_release) - cmd = 'add-apt-repository -y {}'.format(ppa) - _run_with_retries(cmd.split(' ')) - - -def _add_cloud_distro_check(cloud_archive_release, openstack_release): - """Add the cloud pocket, but also check the cloud_archive_release against - the current distro, and use the openstack_release as the full lookup. - - This just calls _add_cloud_pocket() with the openstack_release as pocket - to get the correct cloud-archive.list for dpkg to work with. - - :param cloud_archive_release:String, codename for the distro release. - :param openstack_release: String, spec for the release to look up in the - CLOUD_ARCHIVE_POCKETS - :raises: SourceConfigError if this is the wrong distro, or the pocket spec - doesn't exist. - """ - _verify_is_ubuntu_rel(cloud_archive_release, openstack_release) - _add_cloud_pocket("{}-{}".format(cloud_archive_release, openstack_release)) - - -def _verify_is_ubuntu_rel(release, os_release): - """Verify that the release is in the same as the current ubuntu release. - - :param release: String, lowercase for the release. - :param os_release: String, the os_release being asked for - :raises: SourceConfigError if the release is not the same as the ubuntu - release. - """ - ubuntu_rel = lsb_release()['DISTRIB_CODENAME'] - if release != ubuntu_rel: - raise SourceConfigError( - 'Invalid Cloud Archive release specified: {}-{} on this Ubuntu' - 'version ({})'.format(release, os_release, ubuntu_rel)) - - -def _run_with_retries(cmd, max_retries=CMD_RETRY_COUNT, retry_exitcodes=(1,), - retry_message="", cmd_env=None): - """Run a command and retry until success or max_retries is reached. - - :param: cmd: str: The apt command to run. - :param: max_retries: int: The number of retries to attempt on a fatal - command. Defaults to CMD_RETRY_COUNT. - :param: retry_exitcodes: tuple: Optional additional exit codes to retry. - Defaults to retry on exit code 1. - :param: retry_message: str: Optional log prefix emitted during retries. - :param: cmd_env: dict: Environment variables to add to the command run. - """ - - env = None - kwargs = {} - if cmd_env: - env = os.environ.copy() - env.update(cmd_env) - kwargs['env'] = env - - if not retry_message: - retry_message = "Failed executing '{}'".format(" ".join(cmd)) - retry_message += ". Will retry in {} seconds".format(CMD_RETRY_DELAY) - - retry_count = 0 - result = None - - retry_results = (None,) + retry_exitcodes - while result in retry_results: - try: - # result = subprocess.check_call(cmd, env=env) - result = subprocess.check_call(cmd, **kwargs) - except subprocess.CalledProcessError as e: - retry_count = retry_count + 1 - if retry_count > max_retries: - raise - result = e.returncode - log(retry_message) - time.sleep(CMD_RETRY_DELAY) - - -def _run_apt_command(cmd, fatal=False): - """Run an apt command with optional retries. - - :param: cmd: str: The apt command to run. - :param: fatal: bool: Whether the command's output should be checked and - retried. - """ - # Provide DEBIAN_FRONTEND=noninteractive if not present in the environment. - cmd_env = { - 'DEBIAN_FRONTEND': os.environ.get('DEBIAN_FRONTEND', 'noninteractive')} - - if fatal: - _run_with_retries( - cmd, cmd_env=cmd_env, retry_exitcodes=(1, APT_NO_LOCK,), - retry_message="Couldn't acquire DPKG lock") - else: - env = os.environ.copy() - env.update(cmd_env) - subprocess.call(cmd, env=env) - - -def get_upstream_version(package): - """Determine upstream version based on installed package - - @returns None (if not installed) or the upstream version - """ - import apt_pkg - cache = apt_cache() - try: - pkg = cache[package] - except Exception: - # the package is unknown to the current apt cache. - return None - - if not pkg.current_ver: - # package is known, but no version is currently installed. - return None - - return apt_pkg.upstream_version(pkg.current_ver.ver_str) diff --git a/hooks/charmhelpers/osplatform.py b/hooks/charmhelpers/osplatform.py deleted file mode 100644 index d9a4d5c..0000000 --- a/hooks/charmhelpers/osplatform.py +++ /dev/null @@ -1,25 +0,0 @@ -import platform - - -def get_platform(): - """Return the current OS platform. - - For example: if current os platform is Ubuntu then a string "ubuntu" - will be returned (which is the name of the module). - This string is used to decide which platform module should be imported. - """ - # linux_distribution is deprecated and will be removed in Python 3.7 - # Warings *not* disabled, as we certainly need to fix this. - tuple_platform = platform.linux_distribution() - current_platform = tuple_platform[0] - if "Ubuntu" in current_platform: - return "ubuntu" - elif "CentOS" in current_platform: - return "centos" - elif "debian" in current_platform: - # Stock Python does not detect Ubuntu and instead returns debian. - # Or at least it does in some build environments like Travis CI - return "ubuntu" - else: - raise RuntimeError("This module is not supported on {}." - .format(current_platform)) diff --git a/hooks/charmhelpers/payload/__init__.py b/hooks/charmhelpers/payload/__init__.py deleted file mode 100644 index ee55cb3..0000000 --- a/hooks/charmhelpers/payload/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"Tools for working with files injected into a charm just before deployment." diff --git a/hooks/charmhelpers/payload/execd.py b/hooks/charmhelpers/payload/execd.py deleted file mode 100644 index 1502aa0..0000000 --- a/hooks/charmhelpers/payload/execd.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys -import subprocess -from charmhelpers.core import hookenv - - -def default_execd_dir(): - return os.path.join(os.environ['CHARM_DIR'], 'exec.d') - - -def execd_module_paths(execd_dir=None): - """Generate a list of full paths to modules within execd_dir.""" - if not execd_dir: - execd_dir = default_execd_dir() - - if not os.path.exists(execd_dir): - return - - for subpath in os.listdir(execd_dir): - module = os.path.join(execd_dir, subpath) - if os.path.isdir(module): - yield module - - -def execd_submodule_paths(command, execd_dir=None): - """Generate a list of full paths to the specified command within exec_dir. - """ - for module_path in execd_module_paths(execd_dir): - path = os.path.join(module_path, command) - if os.access(path, os.X_OK) and os.path.isfile(path): - yield path - - -def execd_run(command, execd_dir=None, die_on_error=True, stderr=subprocess.STDOUT): - """Run command for each module within execd_dir which defines it.""" - for submodule_path in execd_submodule_paths(command, execd_dir): - try: - subprocess.check_output(submodule_path, stderr=stderr, - universal_newlines=True) - except subprocess.CalledProcessError as e: - hookenv.log("Error ({}) running {}. Output: {}".format( - e.returncode, e.cmd, e.output)) - if die_on_error: - sys.exit(e.returncode) - - -def execd_preinstall(execd_dir=None): - """Run charm-pre-install for each module within execd_dir.""" - execd_run('charm-pre-install', execd_dir=execd_dir) diff --git a/hooks/client-relation-changed b/hooks/client-relation-changed deleted file mode 120000 index 52d9663..0000000 --- a/hooks/client-relation-changed +++ /dev/null @@ -1 +0,0 @@ -ceph_hooks.py \ No newline at end of file diff --git a/hooks/client-relation-joined b/hooks/client-relation-joined deleted file mode 120000 index 52d9663..0000000 --- a/hooks/client-relation-joined +++ /dev/null @@ -1 +0,0 @@ -ceph_hooks.py \ No newline at end of file diff --git a/hooks/config-changed b/hooks/config-changed deleted file mode 120000 index 52d9663..0000000 --- a/hooks/config-changed +++ /dev/null @@ -1 +0,0 @@ -ceph_hooks.py \ No newline at end of file diff --git a/hooks/install b/hooks/install deleted file mode 100755 index 0bdbf8d..0000000 --- a/hooks/install +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# Wrapper to deal with newer Ubuntu versions that don't have py2 installed -# by default. - -declare -a DEPS=('apt' 'netaddr' 'netifaces' 'pip' 'yaml') - -check_and_install() { - pkg="${1}-${2}" - if ! dpkg -s ${pkg} 2>&1 > /dev/null; then - apt-get -y install ${pkg} - fi -} - -PYTHON="python" - -for dep in ${DEPS[@]}; do - check_and_install ${PYTHON} ${dep} -done - -./hooks/install_deps -exec ./hooks/install.real diff --git a/hooks/install.real b/hooks/install.real deleted file mode 120000 index 52d9663..0000000 --- a/hooks/install.real +++ /dev/null @@ -1 +0,0 @@ -ceph_hooks.py \ No newline at end of file diff --git a/hooks/install_deps b/hooks/install_deps deleted file mode 100755 index da4ba5d..0000000 --- a/hooks/install_deps +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# Wrapper to ensure that python dependencies are installed before we get into -# the python part of the hook execution - -declare -a DEPS=('dnspython' 'pyudev') - -check_and_install() { - pkg="${1}-${2}" - if ! dpkg -s ${pkg} 2>&1 > /dev/null; then - apt-get -y install ${pkg} - fi -} - -PYTHON="python" - -for dep in ${DEPS[@]}; do - check_and_install ${PYTHON} ${dep} -done diff --git a/hooks/mon-relation-changed b/hooks/mon-relation-changed deleted file mode 120000 index 52d9663..0000000 --- a/hooks/mon-relation-changed +++ /dev/null @@ -1 +0,0 @@ -ceph_hooks.py \ No newline at end of file diff --git a/hooks/mon-relation-departed b/hooks/mon-relation-departed deleted file mode 120000 index 52d9663..0000000 --- a/hooks/mon-relation-departed +++ /dev/null @@ -1 +0,0 @@ -ceph_hooks.py \ No newline at end of file diff --git a/hooks/mon-relation-joined b/hooks/mon-relation-joined deleted file mode 120000 index 52d9663..0000000 --- a/hooks/mon-relation-joined +++ /dev/null @@ -1 +0,0 @@ -ceph_hooks.py \ No newline at end of file diff --git a/hooks/nrpe-external-master-relation-changed b/hooks/nrpe-external-master-relation-changed deleted file mode 120000 index 52d9663..0000000 --- a/hooks/nrpe-external-master-relation-changed +++ /dev/null @@ -1 +0,0 @@ -ceph_hooks.py \ No newline at end of file diff --git a/hooks/nrpe-external-master-relation-joined b/hooks/nrpe-external-master-relation-joined deleted file mode 120000 index 52d9663..0000000 --- a/hooks/nrpe-external-master-relation-joined +++ /dev/null @@ -1 +0,0 @@ -ceph_hooks.py \ No newline at end of file diff --git a/hooks/osd-devices-storage-attached b/hooks/osd-devices-storage-attached deleted file mode 120000 index 52d9663..0000000 --- a/hooks/osd-devices-storage-attached +++ /dev/null @@ -1 +0,0 @@ -ceph_hooks.py \ No newline at end of file diff --git a/hooks/osd-devices-storage-detaching b/hooks/osd-devices-storage-detaching deleted file mode 120000 index 52d9663..0000000 --- a/hooks/osd-devices-storage-detaching +++ /dev/null @@ -1 +0,0 @@ -ceph_hooks.py \ No newline at end of file diff --git a/hooks/osd-relation-changed b/hooks/osd-relation-changed deleted file mode 120000 index 52d9663..0000000 --- a/hooks/osd-relation-changed +++ /dev/null @@ -1 +0,0 @@ -ceph_hooks.py \ No newline at end of file diff --git a/hooks/osd-relation-joined b/hooks/osd-relation-joined deleted file mode 120000 index 52d9663..0000000 --- a/hooks/osd-relation-joined +++ /dev/null @@ -1 +0,0 @@ -ceph_hooks.py \ No newline at end of file diff --git a/hooks/radosgw-relation-changed b/hooks/radosgw-relation-changed deleted file mode 120000 index 52d9663..0000000 --- a/hooks/radosgw-relation-changed +++ /dev/null @@ -1 +0,0 @@ -ceph_hooks.py \ No newline at end of file diff --git a/hooks/radosgw-relation-joined b/hooks/radosgw-relation-joined deleted file mode 120000 index 52d9663..0000000 --- a/hooks/radosgw-relation-joined +++ /dev/null @@ -1 +0,0 @@ -ceph_hooks.py \ No newline at end of file diff --git a/hooks/start b/hooks/start deleted file mode 120000 index 52d9663..0000000 --- a/hooks/start +++ /dev/null @@ -1 +0,0 @@ -ceph_hooks.py \ No newline at end of file diff --git a/hooks/stop b/hooks/stop deleted file mode 120000 index 52d9663..0000000 --- a/hooks/stop +++ /dev/null @@ -1 +0,0 @@ -ceph_hooks.py \ No newline at end of file diff --git a/hooks/update-status b/hooks/update-status deleted file mode 120000 index 52d9663..0000000 --- a/hooks/update-status +++ /dev/null @@ -1 +0,0 @@ -ceph_hooks.py \ No newline at end of file diff --git a/hooks/upgrade-charm b/hooks/upgrade-charm deleted file mode 100755 index 6f3d75b..0000000 --- a/hooks/upgrade-charm +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -# Wrapper to ensure that old python bytecode isn't hanging around -# after we upgrade the charm with newer libraries -rm -rf **/*.pyc - -./hooks/install_deps -exec ./hooks/upgrade-charm.real diff --git a/hooks/upgrade-charm.real b/hooks/upgrade-charm.real deleted file mode 120000 index 52d9663..0000000 --- a/hooks/upgrade-charm.real +++ /dev/null @@ -1 +0,0 @@ -ceph_hooks.py \ No newline at end of file diff --git a/hooks/utils.py b/hooks/utils.py deleted file mode 100644 index 68683fc..0000000 --- a/hooks/utils.py +++ /dev/null @@ -1,192 +0,0 @@ -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import socket -import re -from charmhelpers.core.hookenv import ( - unit_get, - cached, - config, - network_get_primary_address, - log, DEBUG, - status_set, -) -from charmhelpers.core import unitdata -from charmhelpers.fetch import ( - apt_install, - filter_installed_packages -) - -from charmhelpers.core.host import ( - lsb_release, - CompareHostReleases, -) - -from charmhelpers.contrib.network.ip import ( - get_address_in_network, - get_ipv6_addr -) - -try: - import dns.resolver -except ImportError: - apt_install(filter_installed_packages(['python-dnspython']), - fatal=True) - import dns.resolver - - -def enable_pocket(pocket): - apt_sources = "/etc/apt/sources.list" - with open(apt_sources, "r") as sources: - lines = sources.readlines() - with open(apt_sources, "w") as sources: - for line in lines: - if pocket in line: - sources.write(re.sub('^# deb', 'deb', line)) - else: - sources.write(line) - - -@cached -def get_unit_hostname(): - return socket.gethostname() - - -@cached -def get_host_ip(hostname=None): - if config('prefer-ipv6'): - return get_ipv6_addr()[0] - - hostname = hostname or unit_get('private-address') - try: - # Test to see if already an IPv4 address - socket.inet_aton(hostname) - return hostname - except socket.error: - # This may throw an NXDOMAIN exception; in which case - # things are badly broken so just let it kill the hook - answers = dns.resolver.query(hostname, 'A') - if answers: - return answers[0].address - - -@cached -def get_public_addr(): - if config('ceph-public-network'): - return get_network_addrs('ceph-public-network')[0] - - try: - return network_get_primary_address('public') - except NotImplementedError: - log("network-get not supported", DEBUG) - - return get_host_ip() - - -@cached -def get_cluster_addr(): - if config('ceph-cluster-network'): - return get_network_addrs('ceph-cluster-network')[0] - - try: - return network_get_primary_address('cluster') - except NotImplementedError: - log("network-get not supported", DEBUG) - - return get_host_ip() - - -def get_networks(config_opt='ceph-public-network'): - """Get all configured networks from provided config option. - - If public network(s) are provided, go through them and return those for - which we have an address configured. - """ - networks = config(config_opt) - if networks: - networks = networks.split() - return [n for n in networks if get_address_in_network(n)] - - return [] - - -def get_network_addrs(config_opt): - """Get all configured public networks addresses. - - If public network(s) are provided, go through them and return the - addresses we have configured on any of those networks. - """ - addrs = [] - networks = config(config_opt) - if networks: - networks = networks.split() - addrs = [get_address_in_network(n) for n in networks] - addrs = [a for a in addrs if a] - - if not addrs: - if networks: - msg = ("Could not find an address on any of '%s' - resolve this " - "error to retry" % (networks)) - status_set('blocked', msg) - raise Exception(msg) - else: - return [get_host_ip()] - - return addrs - - -def assert_charm_supports_ipv6(): - """Check whether we are able to support charms ipv6.""" - _release = lsb_release()['DISTRIB_CODENAME'].lower() - if CompareHostReleases(_release) < "trusty": - raise Exception("IPv6 is not supported in the charms for Ubuntu " - "versions less than Trusty 14.04") - - -# copied charmhelpers.contrib.openstack.utils so that the charm does need the -# entire set of dependencies that that module actually also has to bring in -# from charmhelpers. -def set_unit_paused(): - """Set the unit to a paused state in the local kv() store. - This does NOT actually pause the unit - """ - with unitdata.HookData()() as t: - kv = t[0] - kv.set('unit-paused', True) - - -def clear_unit_paused(): - """Clear the unit from a paused state in the local kv() store - This does NOT actually restart any services - it only clears the - local state. - """ - with unitdata.HookData()() as t: - kv = t[0] - kv.set('unit-paused', False) - - -def is_unit_paused_set(): - """Return the state of the kv().get('unit-paused'). - This does NOT verify that the unit really is paused. - - To help with units that don't have HookData() (testing) - if it excepts, return False - """ - try: - with unitdata.HookData()() as t: - kv = t[0] - # transform something truth-y into a Boolean. - return not(not(kv.get('unit-paused'))) - except: - return False diff --git a/icon.svg b/icon.svg deleted file mode 100644 index e938399..0000000 --- a/icon.svg +++ /dev/null @@ -1,311 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - image/svg+xml - - - - - - - - - - diff --git a/lib/ceph/__init__.py b/lib/ceph/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/lib/ceph/broker.py b/lib/ceph/broker.py deleted file mode 100644 index 95ee779..0000000 --- a/lib/ceph/broker.py +++ /dev/null @@ -1,854 +0,0 @@ -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import collections -import json -import os - -from tempfile import NamedTemporaryFile - -from ceph.utils import ( - get_cephfs, - get_osd_weight -) -from ceph.crush_utils import Crushmap - -from charmhelpers.core.hookenv import ( - log, - DEBUG, - INFO, - ERROR, -) -from charmhelpers.contrib.storage.linux.ceph import ( - create_erasure_profile, - delete_pool, - erasure_profile_exists, - get_osds, - monitor_key_get, - monitor_key_set, - pool_exists, - pool_set, - remove_pool_snapshot, - rename_pool, - set_pool_quota, - snapshot_pool, - validator, - ErasurePool, - Pool, - ReplicatedPool, -) - -# This comes from http://docs.ceph.com/docs/master/rados/operations/pools/ -# This should do a decent job of preventing people from passing in bad values. -# It will give a useful error message -from subprocess import check_call, check_output, CalledProcessError - -POOL_KEYS = { - # "Ceph Key Name": [Python type, [Valid Range]] - "size": [int], - "min_size": [int], - "crash_replay_interval": [int], - "pgp_num": [int], # = or < pg_num - "crush_ruleset": [int], - "hashpspool": [bool], - "nodelete": [bool], - "nopgchange": [bool], - "nosizechange": [bool], - "write_fadvise_dontneed": [bool], - "noscrub": [bool], - "nodeep-scrub": [bool], - "hit_set_type": [str, ["bloom", "explicit_hash", - "explicit_object"]], - "hit_set_count": [int, [1, 1]], - "hit_set_period": [int], - "hit_set_fpp": [float, [0.0, 1.0]], - "cache_target_dirty_ratio": [float], - "cache_target_dirty_high_ratio": [float], - "cache_target_full_ratio": [float], - "target_max_bytes": [int], - "target_max_objects": [int], - "cache_min_flush_age": [int], - "cache_min_evict_age": [int], - "fast_read": [bool], -} - -CEPH_BUCKET_TYPES = [ - 'osd', - 'host', - 'chassis', - 'rack', - 'row', - 'pdu', - 'pod', - 'room', - 'datacenter', - 'region', - 'root' -] - - -def decode_req_encode_rsp(f): - """Decorator to decode incoming requests and encode responses.""" - - def decode_inner(req): - return json.dumps(f(json.loads(req))) - - return decode_inner - - -@decode_req_encode_rsp -def process_requests(reqs): - """Process Ceph broker request(s). - - This is a versioned api. API version must be supplied by the client making - the request. - - :param reqs: dict of request parameters. - :returns: dict. exit-code and reason if not 0 - """ - request_id = reqs.get('request-id') - try: - version = reqs.get('api-version') - if version == 1: - log('Processing request {}'.format(request_id), level=DEBUG) - resp = process_requests_v1(reqs['ops']) - if request_id: - resp['request-id'] = request_id - - return resp - - except Exception as exc: - log(str(exc), level=ERROR) - msg = ("Unexpected error occurred while processing requests: %s" % - reqs) - log(msg, level=ERROR) - return {'exit-code': 1, 'stderr': msg} - - msg = ("Missing or invalid api version ({})".format(version)) - resp = {'exit-code': 1, 'stderr': msg} - if request_id: - resp['request-id'] = request_id - - return resp - - -def handle_create_erasure_profile(request, service): - """Create an erasure profile. - - :param request: dict of request operations and params - :param service: The ceph client to run the command under. - :returns: dict. exit-code and reason if not 0 - """ - # "local" | "shec" or it defaults to "jerasure" - erasure_type = request.get('erasure-type') - # "host" | "rack" or it defaults to "host" # Any valid Ceph bucket - failure_domain = request.get('failure-domain') - name = request.get('name') - k = request.get('k') - m = request.get('m') - l = request.get('l') - - if failure_domain not in CEPH_BUCKET_TYPES: - msg = "failure-domain must be one of {}".format(CEPH_BUCKET_TYPES) - log(msg, level=ERROR) - return {'exit-code': 1, 'stderr': msg} - - create_erasure_profile(service=service, erasure_plugin_name=erasure_type, - profile_name=name, failure_domain=failure_domain, - data_chunks=k, coding_chunks=m, locality=l) - - -def handle_add_permissions_to_key(request, service): - """Groups are defined by the key cephx.groups.(namespace-)?-(name). This - key will contain a dict serialized to JSON with data about the group, - including pools and members. - - A group can optionally have a namespace defined that will be used to - further restrict pool access. - """ - resp = {'exit-code': 0} - - service_name = request.get('name') - group_name = request.get('group') - group_namespace = request.get('group-namespace') - if group_namespace: - group_name = "{}-{}".format(group_namespace, group_name) - group = get_group(group_name=group_name) - service_obj = get_service_groups(service=service_name, - namespace=group_namespace) - format("Service object: {}".format(service_obj)) - permission = request.get('group-permission') or "rwx" - if service_name not in group['services']: - group['services'].append(service_name) - save_group(group=group, group_name=group_name) - if permission not in service_obj['group_names']: - service_obj['group_names'][permission] = [] - if group_name not in service_obj['group_names'][permission]: - service_obj['group_names'][permission].append(group_name) - save_service(service=service_obj, service_name=service_name) - service_obj['groups'] = _build_service_groups(service_obj, - group_namespace) - update_service_permissions(service_name, service_obj, group_namespace) - - return resp - - -def update_service_permissions(service, service_obj=None, namespace=None): - """Update the key permissions for the named client in Ceph""" - if not service_obj: - service_obj = get_service_groups(service=service, namespace=namespace) - permissions = pool_permission_list_for_service(service_obj) - call = ['ceph', 'auth', 'caps', 'client.{}'.format(service)] + permissions - try: - check_call(call) - except CalledProcessError as e: - log("Error updating key capabilities: {}".format(e)) - - -def add_pool_to_group(pool, group, namespace=None): - """Add a named pool to a named group""" - group_name = group - if namespace: - group_name = "{}-{}".format(namespace, group_name) - group = get_group(group_name=group_name) - if pool not in group['pools']: - group["pools"].append(pool) - save_group(group, group_name=group_name) - for service in group['services']: - update_service_permissions(service, namespace=namespace) - - -def pool_permission_list_for_service(service): - """Build the permission string for Ceph for a given service""" - permissions = [] - permission_types = collections.OrderedDict() - for permission, group in service["group_names"].items(): - if permission not in permission_types: - permission_types[permission] = [] - for item in group: - permission_types[permission].append(item) - for permission, groups in permission_types.items(): - permission = "allow {}".format(permission) - for group in groups: - for pool in service['groups'][group]['pools']: - permissions.append("{} pool={}".format(permission, pool)) - return ["mon", "allow r", "osd", ', '.join(permissions)] - - -def get_service_groups(service, namespace=None): - """Services are objects stored with some metadata, they look like (for a - service named "nova"): - { - group_names: {'rwx': ['images']}, - groups: {} - } - After populating the group, it looks like: - { - group_names: {'rwx': ['images']}, - groups: { - 'images': { - pools: ['glance'], - services: ['nova'] - } - } - } - """ - service_json = monitor_key_get(service='admin', - key="cephx.services.{}".format(service)) - try: - service = json.loads(service_json) - except (TypeError, ValueError): - service = None - if service: - service['groups'] = _build_service_groups(service, namespace) - else: - service = {'group_names': {}, 'groups': {}} - return service - - -def _build_service_groups(service, namespace=None): - """Rebuild the 'groups' dict for a service group - - :returns: dict: dictionary keyed by group name of the following - format: - - { - 'images': { - pools: ['glance'], - services: ['nova', 'glance] - }, - 'vms':{ - pools: ['nova'], - services: ['nova'] - } - } - """ - all_groups = {} - for groups in service['group_names'].values(): - for group in groups: - name = group - if namespace: - name = "{}-{}".format(namespace, name) - all_groups[group] = get_group(group_name=name) - return all_groups - - -def get_group(group_name): - """A group is a structure to hold data about a named group, structured as: - { - pools: ['glance'], - services: ['nova'] - } - """ - group_key = get_group_key(group_name=group_name) - group_json = monitor_key_get(service='admin', key=group_key) - try: - group = json.loads(group_json) - except (TypeError, ValueError): - group = None - if not group: - group = { - 'pools': [], - 'services': [] - } - return group - - -def save_service(service_name, service): - """Persist a service in the monitor cluster""" - service['groups'] = {} - return monitor_key_set(service='admin', - key="cephx.services.{}".format(service_name), - value=json.dumps(service)) - - -def save_group(group, group_name): - """Persist a group in the monitor cluster""" - group_key = get_group_key(group_name=group_name) - return monitor_key_set(service='admin', - key=group_key, - value=json.dumps(group)) - - -def get_group_key(group_name): - """Build group key""" - return 'cephx.groups.{}'.format(group_name) - - -def handle_erasure_pool(request, service): - """Create a new erasure coded pool. - - :param request: dict of request operations and params. - :param service: The ceph client to run the command under. - :returns: dict. exit-code and reason if not 0. - """ - pool_name = request.get('name') - erasure_profile = request.get('erasure-profile') - quota = request.get('max-bytes') - weight = request.get('weight') - group_name = request.get('group') - - if erasure_profile is None: - erasure_profile = "default-canonical" - - # Check for missing params - if pool_name is None: - msg = "Missing parameter. name is required for the pool" - log(msg, level=ERROR) - return {'exit-code': 1, 'stderr': msg} - - if group_name: - group_namespace = request.get('group-namespace') - # Add the pool to the group named "group_name" - add_pool_to_group(pool=pool_name, - group=group_name, - namespace=group_namespace) - - # TODO: Default to 3/2 erasure coding. I believe this requires min 5 osds - if not erasure_profile_exists(service=service, name=erasure_profile): - # TODO: Fail and tell them to create the profile or default - msg = ("erasure-profile {} does not exist. Please create it with: " - "create-erasure-profile".format(erasure_profile)) - log(msg, level=ERROR) - return {'exit-code': 1, 'stderr': msg} - - pool = ErasurePool(service=service, name=pool_name, - erasure_code_profile=erasure_profile, - percent_data=weight) - # Ok make the erasure pool - if not pool_exists(service=service, name=pool_name): - log("Creating pool '{}' (erasure_profile={})" - .format(pool.name, erasure_profile), level=INFO) - pool.create() - - # Set a quota if requested - if quota is not None: - set_pool_quota(service=service, pool_name=pool_name, max_bytes=quota) - - -def handle_replicated_pool(request, service): - """Create a new replicated pool. - - :param request: dict of request operations and params. - :param service: The ceph client to run the command under. - :returns: dict. exit-code and reason if not 0. - """ - pool_name = request.get('name') - replicas = request.get('replicas') - quota = request.get('max-bytes') - weight = request.get('weight') - group_name = request.get('group') - - # Optional params - pg_num = request.get('pg_num') - if pg_num: - # Cap pg_num to max allowed just in case. - osds = get_osds(service) - if osds: - pg_num = min(pg_num, (len(osds) * 100 // replicas)) - - # Check for missing params - if pool_name is None or replicas is None: - msg = "Missing parameter. name and replicas are required" - log(msg, level=ERROR) - return {'exit-code': 1, 'stderr': msg} - - if group_name: - group_namespace = request.get('group-namespace') - # Add the pool to the group named "group_name" - add_pool_to_group(pool=pool_name, - group=group_name, - namespace=group_namespace) - - kwargs = {} - if pg_num: - kwargs['pg_num'] = pg_num - if weight: - kwargs['percent_data'] = weight - if replicas: - kwargs['replicas'] = replicas - - pool = ReplicatedPool(service=service, - name=pool_name, **kwargs) - if not pool_exists(service=service, name=pool_name): - log("Creating pool '{}' (replicas={})".format(pool.name, replicas), - level=INFO) - pool.create() - else: - log("Pool '{}' already exists - skipping create".format(pool.name), - level=DEBUG) - - # Set a quota if requested - if quota is not None: - set_pool_quota(service=service, pool_name=pool_name, max_bytes=quota) - - -def handle_create_cache_tier(request, service): - """Create a cache tier on a cold pool. Modes supported are - "writeback" and "readonly". - - :param request: dict of request operations and params - :param service: The ceph client to run the command under. - :returns: dict. exit-code and reason if not 0 - """ - # mode = "writeback" | "readonly" - storage_pool = request.get('cold-pool') - cache_pool = request.get('hot-pool') - cache_mode = request.get('mode') - - if cache_mode is None: - cache_mode = "writeback" - - # cache and storage pool must exist first - if not pool_exists(service=service, name=storage_pool) or not pool_exists( - service=service, name=cache_pool): - msg = ("cold-pool: {} and hot-pool: {} must exist. Please create " - "them first".format(storage_pool, cache_pool)) - log(msg, level=ERROR) - return {'exit-code': 1, 'stderr': msg} - - p = Pool(service=service, name=storage_pool) - p.add_cache_tier(cache_pool=cache_pool, mode=cache_mode) - - -def handle_remove_cache_tier(request, service): - """Remove a cache tier from the cold pool. - - :param request: dict of request operations and params - :param service: The ceph client to run the command under. - :returns: dict. exit-code and reason if not 0 - """ - storage_pool = request.get('cold-pool') - cache_pool = request.get('hot-pool') - # cache and storage pool must exist first - if not pool_exists(service=service, name=storage_pool) or not pool_exists( - service=service, name=cache_pool): - msg = ("cold-pool: {} or hot-pool: {} doesn't exist. Not " - "deleting cache tier".format(storage_pool, cache_pool)) - log(msg, level=ERROR) - return {'exit-code': 1, 'stderr': msg} - - pool = Pool(name=storage_pool, service=service) - pool.remove_cache_tier(cache_pool=cache_pool) - - -def handle_set_pool_value(request, service): - """Sets an arbitrary pool value. - - :param request: dict of request operations and params - :param service: The ceph client to run the command under. - :returns: dict. exit-code and reason if not 0 - """ - # Set arbitrary pool values - params = {'pool': request.get('name'), - 'key': request.get('key'), - 'value': request.get('value')} - if params['key'] not in POOL_KEYS: - msg = "Invalid key '{}'".format(params['key']) - log(msg, level=ERROR) - return {'exit-code': 1, 'stderr': msg} - - # Get the validation method - validator_params = POOL_KEYS[params['key']] - if len(validator_params) is 1: - # Validate that what the user passed is actually legal per Ceph's rules - validator(params['value'], validator_params[0]) - else: - # Validate that what the user passed is actually legal per Ceph's rules - validator(params['value'], validator_params[0], validator_params[1]) - - # Set the value - pool_set(service=service, pool_name=params['pool'], key=params['key'], - value=params['value']) - - -def handle_rgw_regionmap_update(request, service): - """Change the radosgw region map. - - :param request: dict of request operations and params - :param service: The ceph client to run the command under. - :returns: dict. exit-code and reason if not 0 - """ - name = request.get('client-name') - if not name: - msg = "Missing rgw-region or client-name params" - log(msg, level=ERROR) - return {'exit-code': 1, 'stderr': msg} - try: - check_output(['radosgw-admin', - '--id', service, - 'regionmap', 'update', '--name', name]) - except CalledProcessError as err: - log(err.output, level=ERROR) - return {'exit-code': 1, 'stderr': err.output} - - -def handle_rgw_regionmap_default(request, service): - """Create a radosgw region map. - - :param request: dict of request operations and params - :param service: The ceph client to run the command under. - :returns: dict. exit-code and reason if not 0 - """ - region = request.get('rgw-region') - name = request.get('client-name') - if not region or not name: - msg = "Missing rgw-region or client-name params" - log(msg, level=ERROR) - return {'exit-code': 1, 'stderr': msg} - try: - check_output( - [ - 'radosgw-admin', - '--id', service, - 'regionmap', - 'default', - '--rgw-region', region, - '--name', name]) - except CalledProcessError as err: - log(err.output, level=ERROR) - return {'exit-code': 1, 'stderr': err.output} - - -def handle_rgw_zone_set(request, service): - """Create a radosgw zone. - - :param request: dict of request operations and params - :param service: The ceph client to run the command under. - :returns: dict. exit-code and reason if not 0 - """ - json_file = request.get('zone-json') - name = request.get('client-name') - region_name = request.get('region-name') - zone_name = request.get('zone-name') - if not json_file or not name or not region_name or not zone_name: - msg = "Missing json-file or client-name params" - log(msg, level=ERROR) - return {'exit-code': 1, 'stderr': msg} - infile = NamedTemporaryFile(delete=False) - with open(infile.name, 'w') as infile_handle: - infile_handle.write(json_file) - try: - check_output( - [ - 'radosgw-admin', - '--id', service, - 'zone', - 'set', - '--rgw-zone', zone_name, - '--infile', infile.name, - '--name', name, - ] - ) - except CalledProcessError as err: - log(err.output, level=ERROR) - return {'exit-code': 1, 'stderr': err.output} - os.unlink(infile.name) - - -def handle_put_osd_in_bucket(request, service): - """Move an osd into a specified crush bucket. - - :param request: dict of request operations and params - :param service: The ceph client to run the command under. - :returns: dict. exit-code and reason if not 0 - """ - osd_id = request.get('osd') - target_bucket = request.get('bucket') - if not osd_id or not target_bucket: - msg = "Missing OSD ID or Bucket" - log(msg, level=ERROR) - return {'exit-code': 1, 'stderr': msg} - crushmap = Crushmap() - try: - crushmap.ensure_bucket_is_present(target_bucket) - check_output( - [ - 'ceph', - '--id', service, - 'osd', - 'crush', - 'set', - str(osd_id), - str(get_osd_weight(osd_id)), - "root={}".format(target_bucket) - ] - ) - - except Exception as exc: - msg = "Failed to move OSD " \ - "{} into Bucket {} :: {}".format(osd_id, target_bucket, exc) - log(msg, level=ERROR) - return {'exit-code': 1, 'stderr': msg} - - -def handle_rgw_create_user(request, service): - """Create a new rados gateway user. - - :param request: dict of request operations and params - :param service: The ceph client to run the command under. - :returns: dict. exit-code and reason if not 0 - """ - user_id = request.get('rgw-uid') - display_name = request.get('display-name') - name = request.get('client-name') - if not name or not display_name or not user_id: - msg = "Missing client-name, display-name or rgw-uid" - log(msg, level=ERROR) - return {'exit-code': 1, 'stderr': msg} - try: - create_output = check_output( - [ - 'radosgw-admin', - '--id', service, - 'user', - 'create', - '--uid', user_id, - '--display-name', display_name, - '--name', name, - '--system' - ] - ) - try: - user_json = json.loads(str(create_output.decode('UTF-8'))) - return {'exit-code': 0, 'user': user_json} - except ValueError as err: - log(err, level=ERROR) - return {'exit-code': 1, 'stderr': err} - - except CalledProcessError as err: - log(err.output, level=ERROR) - return {'exit-code': 1, 'stderr': err.output} - - -def handle_create_cephfs(request, service): - """Create a new cephfs. - - :param request: The broker request - :param service: The ceph client to run the command under. - :returns: dict. exit-code and reason if not 0 - """ - cephfs_name = request.get('mds_name') - data_pool = request.get('data_pool') - metadata_pool = request.get('metadata_pool') - # Check if the user params were provided - if not cephfs_name or not data_pool or not metadata_pool: - msg = "Missing mds_name, data_pool or metadata_pool params" - log(msg, level=ERROR) - return {'exit-code': 1, 'stderr': msg} - - # Sanity check that the required pools exist - if not pool_exists(service=service, name=data_pool): - msg = "CephFS data pool does not exist. Cannot create CephFS" - log(msg, level=ERROR) - return {'exit-code': 1, 'stderr': msg} - if not pool_exists(service=service, name=metadata_pool): - msg = "CephFS metadata pool does not exist. Cannot create CephFS" - log(msg, level=ERROR) - return {'exit-code': 1, 'stderr': msg} - - if get_cephfs(service=service): - # CephFS new has already been called - log("CephFS already created") - return - - # Finally create CephFS - try: - check_output(["ceph", - '--id', service, - "fs", "new", cephfs_name, - metadata_pool, - data_pool]) - except CalledProcessError as err: - if err.returncode == 22: - log("CephFS already created") - return - else: - log(err.output, level=ERROR) - return {'exit-code': 1, 'stderr': err.output} - - -def handle_rgw_region_set(request, service): - # radosgw-admin region set --infile us.json --name client.radosgw.us-east-1 - """Set the rados gateway region. - - :param request: dict. The broker request. - :param service: The ceph client to run the command under. - :returns: dict. exit-code and reason if not 0 - """ - json_file = request.get('region-json') - name = request.get('client-name') - region_name = request.get('region-name') - zone_name = request.get('zone-name') - if not json_file or not name or not region_name or not zone_name: - msg = "Missing json-file or client-name params" - log(msg, level=ERROR) - return {'exit-code': 1, 'stderr': msg} - infile = NamedTemporaryFile(delete=False) - with open(infile.name, 'w') as infile_handle: - infile_handle.write(json_file) - try: - check_output( - [ - 'radosgw-admin', - '--id', service, - 'region', - 'set', - '--rgw-zone', zone_name, - '--infile', infile.name, - '--name', name, - ] - ) - except CalledProcessError as err: - log(err.output, level=ERROR) - return {'exit-code': 1, 'stderr': err.output} - os.unlink(infile.name) - - -def process_requests_v1(reqs): - """Process v1 requests. - - Takes a list of requests (dicts) and processes each one. If an error is - found, processing stops and the client is notified in the response. - - Returns a response dict containing the exit code (non-zero if any - operation failed along with an explanation). - """ - ret = None - log("Processing {} ceph broker requests".format(len(reqs)), level=INFO) - for req in reqs: - op = req.get('op') - log("Processing op='{}'".format(op), level=DEBUG) - # Use admin client since we do not have other client key locations - # setup to use them for these operations. - svc = 'admin' - if op == "create-pool": - pool_type = req.get('pool-type') # "replicated" | "erasure" - - # Default to replicated if pool_type isn't given - if pool_type == 'erasure': - ret = handle_erasure_pool(request=req, service=svc) - else: - ret = handle_replicated_pool(request=req, service=svc) - elif op == "create-cephfs": - ret = handle_create_cephfs(request=req, service=svc) - elif op == "create-cache-tier": - ret = handle_create_cache_tier(request=req, service=svc) - elif op == "remove-cache-tier": - ret = handle_remove_cache_tier(request=req, service=svc) - elif op == "create-erasure-profile": - ret = handle_create_erasure_profile(request=req, service=svc) - elif op == "delete-pool": - pool = req.get('name') - ret = delete_pool(service=svc, name=pool) - elif op == "rename-pool": - old_name = req.get('name') - new_name = req.get('new-name') - ret = rename_pool(service=svc, old_name=old_name, - new_name=new_name) - elif op == "snapshot-pool": - pool = req.get('name') - snapshot_name = req.get('snapshot-name') - ret = snapshot_pool(service=svc, pool_name=pool, - snapshot_name=snapshot_name) - elif op == "remove-pool-snapshot": - pool = req.get('name') - snapshot_name = req.get('snapshot-name') - ret = remove_pool_snapshot(service=svc, pool_name=pool, - snapshot_name=snapshot_name) - elif op == "set-pool-value": - ret = handle_set_pool_value(request=req, service=svc) - elif op == "rgw-region-set": - ret = handle_rgw_region_set(request=req, service=svc) - elif op == "rgw-zone-set": - ret = handle_rgw_zone_set(request=req, service=svc) - elif op == "rgw-regionmap-update": - ret = handle_rgw_regionmap_update(request=req, service=svc) - elif op == "rgw-regionmap-default": - ret = handle_rgw_regionmap_default(request=req, service=svc) - elif op == "rgw-create-user": - ret = handle_rgw_create_user(request=req, service=svc) - elif op == "move-osd-to-bucket": - ret = handle_put_osd_in_bucket(request=req, service=svc) - elif op == "add-permissions-to-key": - ret = handle_add_permissions_to_key(request=req, service=svc) - else: - msg = "Unknown operation '{}'".format(op) - log(msg, level=ERROR) - return {'exit-code': 1, 'stderr': msg} - - if type(ret) == dict and 'exit-code' in ret: - return ret - - return {'exit-code': 0} diff --git a/lib/ceph/crush_utils.py b/lib/ceph/crush_utils.py deleted file mode 100644 index 8b6876c..0000000 --- a/lib/ceph/crush_utils.py +++ /dev/null @@ -1,154 +0,0 @@ -# Copyright 2014 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -from subprocess import check_output, CalledProcessError - -from charmhelpers.core.hookenv import ( - log, - ERROR, -) - -CRUSH_BUCKET = """root {name} {{ - id {id} # do not change unnecessarily - # weight 0.000 - alg straw - hash 0 # rjenkins1 -}} - -rule {name} {{ - ruleset 0 - type replicated - min_size 1 - max_size 10 - step take {name} - step chooseleaf firstn 0 type host - step emit -}}""" - -# This regular expression looks for a string like: -# root NAME { -# id NUMBER -# so that we can extract NAME and ID from the crushmap -CRUSHMAP_BUCKETS_RE = re.compile(r"root\s+(.+)\s+\{\s*id\s+(-?\d+)") - -# This regular expression looks for ID strings in the crushmap like: -# id NUMBER -# so that we can extract the IDs from a crushmap -CRUSHMAP_ID_RE = re.compile(r"id\s+(-?\d+)") - - -class Crushmap(object): - """An object oriented approach to Ceph crushmap management.""" - - def __init__(self): - self._crushmap = self.load_crushmap() - roots = re.findall(CRUSHMAP_BUCKETS_RE, self._crushmap) - buckets = [] - ids = list(map( - lambda x: int(x), - re.findall(CRUSHMAP_ID_RE, self._crushmap))) - ids = sorted(ids) - if roots != []: - for root in roots: - buckets.append(CRUSHBucket(root[0], root[1], True)) - - self._buckets = buckets - if ids != []: - self._ids = ids - else: - self._ids = [0] - - def load_crushmap(self): - try: - crush = str(check_output(['ceph', 'osd', 'getcrushmap']) - .decode('UTF-8')) - return str(check_output(['crushtool', '-d', '-'], - stdin=crush.stdout) - .decode('UTF-8')) - except CalledProcessError as e: - log("Error occured while loading and decompiling CRUSH map:" - "{}".format(e), ERROR) - raise "Failed to read CRUSH map" - - def ensure_bucket_is_present(self, bucket_name): - if bucket_name not in [bucket.name for bucket in self.buckets()]: - self.add_bucket(bucket_name) - self.save() - - def buckets(self): - """Return a list of buckets that are in the Crushmap.""" - return self._buckets - - def add_bucket(self, bucket_name): - """Add a named bucket to Ceph""" - new_id = min(self._ids) - 1 - self._ids.append(new_id) - self._buckets.append(CRUSHBucket(bucket_name, new_id)) - - def save(self): - """Persist Crushmap to Ceph""" - try: - crushmap = self.build_crushmap() - compiled = str(check_output(['crushtool', '-c', '/dev/stdin', '-o', - '/dev/stdout'], stdin=crushmap) - .decode('UTF-8')) - ceph_output = str(check_output(['ceph', 'osd', 'setcrushmap', '-i', - '/dev/stdin'], stdin=compiled) - .decode('UTF-8')) - return ceph_output - except CalledProcessError as e: - log("save error: {}".format(e)) - raise "Failed to save CRUSH map." - - def build_crushmap(self): - """Modifies the current CRUSH map to include the new buckets""" - tmp_crushmap = self._crushmap - for bucket in self._buckets: - if not bucket.default: - tmp_crushmap = "{}\n\n{}".format( - tmp_crushmap, - Crushmap.bucket_string(bucket.name, bucket.id)) - - return tmp_crushmap - - @staticmethod - def bucket_string(name, id): - return CRUSH_BUCKET.format(name=name, id=id) - - -class CRUSHBucket(object): - """CRUSH bucket description object.""" - - def __init__(self, name, id, default=False): - self.name = name - self.id = int(id) - self.default = default - - def __repr__(self): - return "Bucket {{Name: {name}, ID: {id}}}".format( - name=self.name, id=self.id) - - def __eq__(self, other): - """Override the default Equals behavior""" - if isinstance(other, self.__class__): - return self.__dict__ == other.__dict__ - return NotImplemented - - def __ne__(self, other): - """Define a non-equality test""" - if isinstance(other, self.__class__): - return not self.__eq__(other) - return NotImplemented diff --git a/lib/ceph/utils.py b/lib/ceph/utils.py deleted file mode 100644 index 7665630..0000000 --- a/lib/ceph/utils.py +++ /dev/null @@ -1,2316 +0,0 @@ -# Copyright 2017 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import collections -import ctypes -import errno -import json -import os -import pyudev -import random -import re -import socket -import subprocess -import sys -import time -import shutil - -from datetime import datetime - -from charmhelpers.core import hookenv -from charmhelpers.core import templating -from charmhelpers.core.decorators import retry_on_exception -from charmhelpers.core.host import ( - chownr, - cmp_pkgrevno, - lsb_release, - mkdir, - mounts, - owner, - service_restart, - service_start, - service_stop, - CompareHostReleases, - is_container, -) -from charmhelpers.core.hookenv import ( - cached, - config, - log, - status_set, - DEBUG, - ERROR, - WARNING, - storage_get, - storage_list, -) -from charmhelpers.fetch import ( - apt_cache, - add_source, apt_install, apt_update -) -from charmhelpers.contrib.storage.linux.ceph import ( - get_mon_map, - monitor_key_set, - monitor_key_exists, - monitor_key_get, -) -from charmhelpers.contrib.storage.linux.utils import ( - is_block_device, - is_device_mounted, - zap_disk, -) -from charmhelpers.contrib.openstack.utils import ( - get_os_codename_install_source, -) - -CEPH_BASE_DIR = os.path.join(os.sep, 'var', 'lib', 'ceph') -OSD_BASE_DIR = os.path.join(CEPH_BASE_DIR, 'osd') -HDPARM_FILE = os.path.join(os.sep, 'etc', 'hdparm.conf') - -LEADER = 'leader' -PEON = 'peon' -QUORUM = [LEADER, PEON] - -PACKAGES = ['ceph', 'gdisk', 'ntp', 'btrfs-tools', 'python-ceph', - 'radosgw', 'xfsprogs', 'python-pyudev'] - -LinkSpeed = { - "BASE_10": 10, - "BASE_100": 100, - "BASE_1000": 1000, - "GBASE_10": 10000, - "GBASE_40": 40000, - "GBASE_100": 100000, - "UNKNOWN": None -} - -# Mapping of adapter speed to sysctl settings -NETWORK_ADAPTER_SYSCTLS = { - # 10Gb - LinkSpeed["GBASE_10"]: { - 'net.core.rmem_default': 524287, - 'net.core.wmem_default': 524287, - 'net.core.rmem_max': 524287, - 'net.core.wmem_max': 524287, - 'net.core.optmem_max': 524287, - 'net.core.netdev_max_backlog': 300000, - 'net.ipv4.tcp_rmem': '10000000 10000000 10000000', - 'net.ipv4.tcp_wmem': '10000000 10000000 10000000', - 'net.ipv4.tcp_mem': '10000000 10000000 10000000' - }, - # Mellanox 10/40Gb - LinkSpeed["GBASE_40"]: { - 'net.ipv4.tcp_timestamps': 0, - 'net.ipv4.tcp_sack': 1, - 'net.core.netdev_max_backlog': 250000, - 'net.core.rmem_max': 4194304, - 'net.core.wmem_max': 4194304, - 'net.core.rmem_default': 4194304, - 'net.core.wmem_default': 4194304, - 'net.core.optmem_max': 4194304, - 'net.ipv4.tcp_rmem': '4096 87380 4194304', - 'net.ipv4.tcp_wmem': '4096 65536 4194304', - 'net.ipv4.tcp_low_latency': 1, - 'net.ipv4.tcp_adv_win_scale': 1 - } -} - - -class Partition(object): - def __init__(self, name, number, size, start, end, sectors, uuid): - """A block device partition. - - :param name: Name of block device - :param number: Partition number - :param size: Capacity of the device - :param start: Starting block - :param end: Ending block - :param sectors: Number of blocks - :param uuid: UUID of the partition - """ - self.name = name, - self.number = number - self.size = size - self.start = start - self.end = end - self.sectors = sectors - self.uuid = uuid - - def __str__(self): - return "number: {} start: {} end: {} sectors: {} size: {} " \ - "name: {} uuid: {}".format(self.number, self.start, - self.end, - self.sectors, self.size, - self.name, self.uuid) - - def __eq__(self, other): - if isinstance(other, self.__class__): - return self.__dict__ == other.__dict__ - return False - - def __ne__(self, other): - return not self.__eq__(other) - - -def unmounted_disks(): - """List of unmounted block devices on the current host.""" - disks = [] - context = pyudev.Context() - for device in context.list_devices(DEVTYPE='disk'): - if device['SUBSYSTEM'] == 'block': - matched = False - for block_type in [u'dm', u'loop', u'ram', u'nbd']: - if block_type in device.device_node: - matched = True - if matched: - continue - disks.append(device.device_node) - log("Found disks: {}".format(disks)) - return [disk for disk in disks if not is_device_mounted(disk)] - - -def save_sysctls(sysctl_dict, save_location): - """Persist the sysctls to the hard drive. - - :param sysctl_dict: dict - :param save_location: path to save the settings to - :raises: IOError if anything goes wrong with writing. - """ - try: - # Persist the settings for reboots - with open(save_location, "w") as fd: - for key, value in sysctl_dict.items(): - fd.write("{}={}\n".format(key, value)) - - except IOError as e: - log("Unable to persist sysctl settings to {}. Error {}".format( - save_location, e), level=ERROR) - raise - - -def tune_nic(network_interface): - """This will set optimal sysctls for the particular network adapter. - - :param network_interface: string The network adapter name. - """ - speed = get_link_speed(network_interface) - if speed in NETWORK_ADAPTER_SYSCTLS: - status_set('maintenance', 'Tuning device {}'.format( - network_interface)) - sysctl_file = os.path.join( - os.sep, - 'etc', - 'sysctl.d', - '51-ceph-osd-charm-{}.conf'.format(network_interface)) - try: - log("Saving sysctl_file: {} values: {}".format( - sysctl_file, NETWORK_ADAPTER_SYSCTLS[speed]), - level=DEBUG) - save_sysctls(sysctl_dict=NETWORK_ADAPTER_SYSCTLS[speed], - save_location=sysctl_file) - except IOError as e: - log("Write to /etc/sysctl.d/51-ceph-osd-charm-{} " - "failed. {}".format(network_interface, e), - level=ERROR) - - try: - # Apply the settings - log("Applying sysctl settings", level=DEBUG) - subprocess.check_output(["sysctl", "-p", sysctl_file]) - except subprocess.CalledProcessError as err: - log('sysctl -p {} failed with error {}'.format(sysctl_file, - err.output), - level=ERROR) - else: - log("No settings found for network adapter: {}".format( - network_interface), level=DEBUG) - - -def get_link_speed(network_interface): - """This will find the link speed for a given network device. Returns None - if an error occurs. - :param network_interface: string The network adapter interface. - :returns: LinkSpeed - """ - speed_path = os.path.join(os.sep, 'sys', 'class', 'net', - network_interface, 'speed') - # I'm not sure where else we'd check if this doesn't exist - if not os.path.exists(speed_path): - return LinkSpeed["UNKNOWN"] - - try: - with open(speed_path, 'r') as sysfs: - nic_speed = sysfs.readlines() - - # Did we actually read anything? - if not nic_speed: - return LinkSpeed["UNKNOWN"] - - # Try to find a sysctl match for this particular speed - for name, speed in LinkSpeed.items(): - if speed == int(nic_speed[0].strip()): - return speed - # Default to UNKNOWN if we can't find a match - return LinkSpeed["UNKNOWN"] - except IOError as e: - log("Unable to open {path} because of error: {error}".format( - path=speed_path, - error=e), level='error') - return LinkSpeed["UNKNOWN"] - - -def persist_settings(settings_dict): - # Write all settings to /etc/hdparm.conf - """ This will persist the hard drive settings to the /etc/hdparm.conf file - - The settings_dict should be in the form of {"uuid": {"key":"value"}} - - :param settings_dict: dict of settings to save - """ - if not settings_dict: - return - - try: - templating.render(source='hdparm.conf', target=HDPARM_FILE, - context=settings_dict) - except IOError as err: - log("Unable to open {path} because of error: {error}".format( - path=HDPARM_FILE, error=err), level=ERROR) - except Exception as e: - # The templating.render can raise a jinja2 exception if the - # template is not found. Rather than polluting the import - # space of this charm, simply catch Exception - log('Unable to render {path} due to error: {error}'.format( - path=HDPARM_FILE, error=e), level=ERROR) - - -def set_max_sectors_kb(dev_name, max_sectors_size): - """This function sets the max_sectors_kb size of a given block device. - - :param dev_name: Name of the block device to query - :param max_sectors_size: int of the max_sectors_size to save - """ - max_sectors_kb_path = os.path.join('sys', 'block', dev_name, 'queue', - 'max_sectors_kb') - try: - with open(max_sectors_kb_path, 'w') as f: - f.write(max_sectors_size) - except IOError as e: - log('Failed to write max_sectors_kb to {}. Error: {}'.format( - max_sectors_kb_path, e), level=ERROR) - - -def get_max_sectors_kb(dev_name): - """This function gets the max_sectors_kb size of a given block device. - - :param dev_name: Name of the block device to query - :returns: int which is either the max_sectors_kb or 0 on error. - """ - max_sectors_kb_path = os.path.join('sys', 'block', dev_name, 'queue', - 'max_sectors_kb') - - # Read in what Linux has set by default - if os.path.exists(max_sectors_kb_path): - try: - with open(max_sectors_kb_path, 'r') as f: - max_sectors_kb = f.read().strip() - return int(max_sectors_kb) - except IOError as e: - log('Failed to read max_sectors_kb to {}. Error: {}'.format( - max_sectors_kb_path, e), level=ERROR) - # Bail. - return 0 - return 0 - - -def get_max_hw_sectors_kb(dev_name): - """This function gets the max_hw_sectors_kb for a given block device. - - :param dev_name: Name of the block device to query - :returns: int which is either the max_hw_sectors_kb or 0 on error. - """ - max_hw_sectors_kb_path = os.path.join('sys', 'block', dev_name, 'queue', - 'max_hw_sectors_kb') - # Read in what the hardware supports - if os.path.exists(max_hw_sectors_kb_path): - try: - with open(max_hw_sectors_kb_path, 'r') as f: - max_hw_sectors_kb = f.read().strip() - return int(max_hw_sectors_kb) - except IOError as e: - log('Failed to read max_hw_sectors_kb to {}. Error: {}'.format( - max_hw_sectors_kb_path, e), level=ERROR) - return 0 - return 0 - - -def set_hdd_read_ahead(dev_name, read_ahead_sectors=256): - """This function sets the hard drive read ahead. - - :param dev_name: Name of the block device to set read ahead on. - :param read_ahead_sectors: int How many sectors to read ahead. - """ - try: - # Set the read ahead sectors to 256 - log('Setting read ahead to {} for device {}'.format( - read_ahead_sectors, - dev_name)) - subprocess.check_output(['hdparm', - '-a{}'.format(read_ahead_sectors), - dev_name]) - except subprocess.CalledProcessError as e: - log('hdparm failed with error: {}'.format(e.output), - level=ERROR) - - -def get_block_uuid(block_dev): - """This queries blkid to get the uuid for a block device. - - :param block_dev: Name of the block device to query. - :returns: The UUID of the device or None on Error. - """ - try: - block_info = str(subprocess - .check_output(['blkid', '-o', 'export', block_dev]) - .decode('UTF-8')) - for tag in block_info.split('\n'): - parts = tag.split('=') - if parts[0] == 'UUID': - return parts[1] - return None - except subprocess.CalledProcessError as err: - log('get_block_uuid failed with error: {}'.format(err.output), - level=ERROR) - return None - - -def check_max_sectors(save_settings_dict, - block_dev, - uuid): - """Tune the max_hw_sectors if needed. - - make sure that /sys/.../max_sectors_kb matches max_hw_sectors_kb or at - least 1MB for spinning disks - If the box has a RAID card with cache this could go much bigger. - - :param save_settings_dict: The dict used to persist settings - :param block_dev: A block device name: Example: /dev/sda - :param uuid: The uuid of the block device - """ - dev_name = None - path_parts = os.path.split(block_dev) - if len(path_parts) == 2: - dev_name = path_parts[1] - else: - log('Unable to determine the block device name from path: {}'.format( - block_dev)) - # Play it safe and bail - return - max_sectors_kb = get_max_sectors_kb(dev_name=dev_name) - max_hw_sectors_kb = get_max_hw_sectors_kb(dev_name=dev_name) - - if max_sectors_kb < max_hw_sectors_kb: - # OK we have a situation where the hardware supports more than Linux is - # currently requesting - config_max_sectors_kb = hookenv.config('max-sectors-kb') - if config_max_sectors_kb < max_hw_sectors_kb: - # Set the max_sectors_kb to the config.yaml value if it is less - # than the max_hw_sectors_kb - log('Setting max_sectors_kb for device {} to {}'.format( - dev_name, config_max_sectors_kb)) - save_settings_dict[ - "drive_settings"][uuid][ - "read_ahead_sect"] = config_max_sectors_kb - set_max_sectors_kb(dev_name=dev_name, - max_sectors_size=config_max_sectors_kb) - else: - # Set to the max_hw_sectors_kb - log('Setting max_sectors_kb for device {} to {}'.format( - dev_name, max_hw_sectors_kb)) - save_settings_dict[ - "drive_settings"][uuid]['read_ahead_sect'] = max_hw_sectors_kb - set_max_sectors_kb(dev_name=dev_name, - max_sectors_size=max_hw_sectors_kb) - else: - log('max_sectors_kb match max_hw_sectors_kb. No change needed for ' - 'device: {}'.format(block_dev)) - - -def tune_dev(block_dev): - """Try to make some intelligent decisions with HDD tuning. Future work will - include optimizing SSDs. - - This function will change the read ahead sectors and the max write - sectors for each block device. - - :param block_dev: A block device name: Example: /dev/sda - """ - uuid = get_block_uuid(block_dev) - if uuid is None: - log('block device {} uuid is None. Unable to save to ' - 'hdparm.conf'.format(block_dev), level=DEBUG) - return - save_settings_dict = {} - log('Tuning device {}'.format(block_dev)) - status_set('maintenance', 'Tuning device {}'.format(block_dev)) - set_hdd_read_ahead(block_dev) - save_settings_dict["drive_settings"] = {} - save_settings_dict["drive_settings"][uuid] = {} - save_settings_dict["drive_settings"][uuid]['read_ahead_sect'] = 256 - - check_max_sectors(block_dev=block_dev, - save_settings_dict=save_settings_dict, - uuid=uuid) - - persist_settings(settings_dict=save_settings_dict) - status_set('maintenance', 'Finished tuning device {}'.format(block_dev)) - - -def ceph_user(): - if get_version() > 1: - return 'ceph' - else: - return "root" - - -class CrushLocation(object): - def __init__(self, - name, - identifier, - host, - rack, - row, - datacenter, - chassis, - root): - self.name = name - self.identifier = identifier - self.host = host - self.rack = rack - self.row = row - self.datacenter = datacenter - self.chassis = chassis - self.root = root - - def __str__(self): - return "name: {} id: {} host: {} rack: {} row: {} datacenter: {} " \ - "chassis :{} root: {}".format(self.name, self.identifier, - self.host, self.rack, self.row, - self.datacenter, self.chassis, - self.root) - - def __eq__(self, other): - return not self.name < other.name and not other.name < self.name - - def __ne__(self, other): - return self.name < other.name or other.name < self.name - - def __gt__(self, other): - return self.name > other.name - - def __ge__(self, other): - return not self.name < other.name - - def __le__(self, other): - return self.name < other.name - - -def get_osd_weight(osd_id): - """Returns the weight of the specified OSD. - - :returns: Float - :raises: ValueError if the monmap fails to parse. - :raises: CalledProcessError if our ceph command fails. - """ - try: - tree = str(subprocess - .check_output(['ceph', 'osd', 'tree', '--format=json']) - .decode('UTF-8')) - try: - json_tree = json.loads(tree) - # Make sure children are present in the json - if not json_tree['nodes']: - return None - for device in json_tree['nodes']: - if device['type'] == 'osd' and device['name'] == osd_id: - return device['crush_weight'] - except ValueError as v: - log("Unable to parse ceph tree json: {}. Error: {}".format( - tree, v)) - raise - except subprocess.CalledProcessError as e: - log("ceph osd tree command failed with message: {}".format( - e)) - raise - - -def get_osd_tree(service): - """Returns the current osd map in JSON. - - :returns: List. - :raises: ValueError if the monmap fails to parse. - Also raises CalledProcessError if our ceph command fails - """ - try: - tree = str(subprocess - .check_output(['ceph', '--id', service, - 'osd', 'tree', '--format=json']) - .decode('UTF-8')) - try: - json_tree = json.loads(tree) - crush_list = [] - # Make sure children are present in the json - if not json_tree['nodes']: - return None - child_ids = json_tree['nodes'][0]['children'] - for child in json_tree['nodes']: - if child['id'] in child_ids: - crush_list.append( - CrushLocation( - name=child.get('name'), - identifier=child['id'], - host=child.get('host'), - rack=child.get('rack'), - row=child.get('row'), - datacenter=child.get('datacenter'), - chassis=child.get('chassis'), - root=child.get('root') - ) - ) - return crush_list - except ValueError as v: - log("Unable to parse ceph tree json: {}. Error: {}".format( - tree, v)) - raise - except subprocess.CalledProcessError as e: - log("ceph osd tree command failed with message: {}".format( - e)) - raise - - -def _get_child_dirs(path): - """Returns a list of directory names in the specified path. - - :param path: a full path listing of the parent directory to return child - directory names - :returns: list. A list of child directories under the parent directory - :raises: ValueError if the specified path does not exist or is not a - directory, - OSError if an error occurs reading the directory listing - """ - if not os.path.exists(path): - raise ValueError('Specfied path "%s" does not exist' % path) - if not os.path.isdir(path): - raise ValueError('Specified path "%s" is not a directory' % path) - - files_in_dir = [os.path.join(path, f) for f in os.listdir(path)] - return list(filter(os.path.isdir, files_in_dir)) - - -def _get_osd_num_from_dirname(dirname): - """Parses the dirname and returns the OSD id. - - Parses a string in the form of 'ceph-{osd#}' and returns the osd number - from the directory name. - - :param dirname: the directory name to return the OSD number from - :return int: the osd number the directory name corresponds to - :raises ValueError: if the osd number cannot be parsed from the provided - directory name. - """ - match = re.search('ceph-(?P\d+)', dirname) - if not match: - raise ValueError("dirname not in correct format: {}".format(dirname)) - - return match.group('osd_id') - - -def get_local_osd_ids(): - """This will list the /var/lib/ceph/osd/* directories and try - to split the ID off of the directory name and return it in - a list. - - :returns: list. A list of osd identifiers - :raises: OSError if something goes wrong with listing the directory. - """ - osd_ids = [] - osd_path = os.path.join(os.sep, 'var', 'lib', 'ceph', 'osd') - if os.path.exists(osd_path): - try: - dirs = os.listdir(osd_path) - for osd_dir in dirs: - osd_id = osd_dir.split('-')[1] - if _is_int(osd_id): - osd_ids.append(osd_id) - except OSError: - raise - return osd_ids - - -def get_local_mon_ids(): - """This will list the /var/lib/ceph/mon/* directories and try - to split the ID off of the directory name and return it in - a list. - - :returns: list. A list of monitor identifiers - :raises: OSError if something goes wrong with listing the directory. - """ - mon_ids = [] - mon_path = os.path.join(os.sep, 'var', 'lib', 'ceph', 'mon') - if os.path.exists(mon_path): - try: - dirs = os.listdir(mon_path) - for mon_dir in dirs: - # Basically this takes everything after ceph- as the monitor ID - match = re.search('ceph-(?P.*)', mon_dir) - if match: - mon_ids.append(match.group('mon_id')) - except OSError: - raise - return mon_ids - - -def _is_int(v): - """Return True if the object v can be turned into an integer.""" - try: - int(v) - return True - except ValueError: - return False - - -def get_version(): - """Derive Ceph release from an installed package.""" - import apt_pkg as apt - - cache = apt_cache() - package = "ceph" - try: - pkg = cache[package] - except: - # the package is unknown to the current apt cache. - e = 'Could not determine version of package with no installation ' \ - 'candidate: %s' % package - error_out(e) - - if not pkg.current_ver: - # package is known, but no version is currently installed. - e = 'Could not determine version of uninstalled package: %s' % package - error_out(e) - - vers = apt.upstream_version(pkg.current_ver.ver_str) - - # x.y match only for 20XX.X - # and ignore patch level for other packages - match = re.match('^(\d+)\.(\d+)', vers) - - if match: - vers = match.group(0) - return float(vers) - - -def error_out(msg): - log("FATAL ERROR: {}".format(msg), - level=ERROR) - sys.exit(1) - - -def is_quorum(): - asok = "/var/run/ceph/ceph-mon.{}.asok".format(socket.gethostname()) - cmd = [ - "sudo", - "-u", - ceph_user(), - "ceph", - "--admin-daemon", - asok, - "mon_status" - ] - if os.path.exists(asok): - try: - result = json.loads(str(subprocess - .check_output(cmd) - .decode('UTF-8'))) - except subprocess.CalledProcessError: - return False - except ValueError: - # Non JSON response from mon_status - return False - if result['state'] in QUORUM: - return True - else: - return False - else: - return False - - -def is_leader(): - asok = "/var/run/ceph/ceph-mon.{}.asok".format(socket.gethostname()) - cmd = [ - "sudo", - "-u", - ceph_user(), - "ceph", - "--admin-daemon", - asok, - "mon_status" - ] - if os.path.exists(asok): - try: - result = json.loads(str(subprocess - .check_output(cmd) - .decode('UTF-8'))) - except subprocess.CalledProcessError: - return False - except ValueError: - # Non JSON response from mon_status - return False - if result['state'] == LEADER: - return True - else: - return False - else: - return False - - -def wait_for_quorum(): - while not is_quorum(): - log("Waiting for quorum to be reached") - time.sleep(3) - - -def add_bootstrap_hint(peer): - asok = "/var/run/ceph/ceph-mon.{}.asok".format(socket.gethostname()) - cmd = [ - "sudo", - "-u", - ceph_user(), - "ceph", - "--admin-daemon", - asok, - "add_bootstrap_peer_hint", - peer - ] - if os.path.exists(asok): - # Ignore any errors for this call - subprocess.call(cmd) - - -DISK_FORMATS = [ - 'xfs', - 'ext4', - 'btrfs' -] - -CEPH_PARTITIONS = [ - '89C57F98-2FE5-4DC0-89C1-5EC00CEFF2BE', # ceph encrypted disk in creation - '45B0969E-9B03-4F30-B4C6-5EC00CEFF106', # ceph encrypted journal - '4FBD7E29-9D25-41B8-AFD0-5EC00CEFF05D', # ceph encrypted osd data - '4FBD7E29-9D25-41B8-AFD0-062C0CEFF05D', # ceph osd data - '45B0969E-9B03-4F30-B4C6-B4B80CEFF106', # ceph osd journal - '89C57F98-2FE5-4DC0-89C1-F3AD0CEFF2BE', # ceph disk in creation -] - - -def umount(mount_point): - """This function unmounts a mounted directory forcibly. This will - be used for unmounting broken hard drive mounts which may hang. - - If umount returns EBUSY this will lazy unmount. - - :param mount_point: str. A String representing the filesystem mount point - :returns: int. Returns 0 on success. errno otherwise. - """ - libc_path = ctypes.util.find_library("c") - libc = ctypes.CDLL(libc_path, use_errno=True) - - # First try to umount with MNT_FORCE - ret = libc.umount(mount_point, 1) - if ret < 0: - err = ctypes.get_errno() - if err == errno.EBUSY: - # Detach from try. IE lazy umount - ret = libc.umount(mount_point, 2) - if ret < 0: - err = ctypes.get_errno() - return err - return 0 - else: - return err - return 0 - - -def replace_osd(dead_osd_number, - dead_osd_device, - new_osd_device, - osd_format, - osd_journal, - reformat_osd=False, - ignore_errors=False): - """This function will automate the replacement of a failed osd disk as much - as possible. It will revoke the keys for the old osd, remove it from the - crush map and then add a new osd into the cluster. - - :param dead_osd_number: The osd number found in ceph osd tree. Example: 99 - :param dead_osd_device: The physical device. Example: /dev/sda - :param osd_format: - :param osd_journal: - :param reformat_osd: - :param ignore_errors: - """ - host_mounts = mounts() - mount_point = None - for mount in host_mounts: - if mount[1] == dead_osd_device: - mount_point = mount[0] - # need to convert dev to osd number - # also need to get the mounted drive so we can tell the admin to - # replace it - try: - # Drop this osd out of the cluster. This will begin a - # rebalance operation - status_set('maintenance', 'Removing osd {}'.format(dead_osd_number)) - subprocess.check_output([ - 'ceph', - '--id', - 'osd-upgrade', - 'osd', 'out', - 'osd.{}'.format(dead_osd_number)]) - - # Kill the osd process if it's not already dead - if systemd(): - service_stop('ceph-osd@{}'.format(dead_osd_number)) - else: - subprocess.check_output(['stop', 'ceph-osd', 'id={}'.format( - dead_osd_number)]) - # umount if still mounted - ret = umount(mount_point) - if ret < 0: - raise RuntimeError('umount {} failed with error: {}'.format( - mount_point, os.strerror(ret))) - # Clean up the old mount point - shutil.rmtree(mount_point) - subprocess.check_output([ - 'ceph', - '--id', - 'osd-upgrade', - 'osd', 'crush', 'remove', - 'osd.{}'.format(dead_osd_number)]) - # Revoke the OSDs access keys - subprocess.check_output([ - 'ceph', - '--id', - 'osd-upgrade', - 'auth', 'del', - 'osd.{}'.format(dead_osd_number)]) - subprocess.check_output([ - 'ceph', - '--id', - 'osd-upgrade', - 'osd', 'rm', - 'osd.{}'.format(dead_osd_number)]) - status_set('maintenance', 'Setting up replacement osd {}'.format( - new_osd_device)) - osdize(new_osd_device, - osd_format, - osd_journal, - reformat_osd, - ignore_errors) - except subprocess.CalledProcessError as e: - log('replace_osd failed with error: ' + e.output) - - -def get_partition_list(dev): - """Lists the partitions of a block device. - - :param dev: Path to a block device. ex: /dev/sda - :returns: Returns a list of Partition objects. - :raises: CalledProcessException if lsblk fails - """ - partitions_list = [] - try: - partitions = get_partitions(dev) - # For each line of output - for partition in partitions: - parts = partition.split() - try: - partitions_list.append( - Partition(number=parts[0], - start=parts[1], - end=parts[2], - sectors=parts[3], - size=parts[4], - name=parts[5], - uuid=parts[6]) - ) - except IndexError: - partitions_list.append( - Partition(number=parts[0], - start=parts[1], - end=parts[2], - sectors=parts[3], - size=parts[4], - name="", - uuid=parts[5]) - ) - - return partitions_list - except subprocess.CalledProcessError: - raise - - -def is_osd_disk(dev): - partitions = get_partition_list(dev) - for partition in partitions: - try: - info = str(subprocess - .check_output(['sgdisk', '-i', partition.number, dev]) - .decode('UTF-8')) - info = info.split("\n") # IGNORE:E1103 - for line in info: - for ptype in CEPH_PARTITIONS: - sig = 'Partition GUID code: {}'.format(ptype) - if line.startswith(sig): - return True - except subprocess.CalledProcessError as e: - log("sgdisk inspection of partition {} on {} failed with " - "error: {}. Skipping".format(partition.minor, dev, e), - level=ERROR) - return False - - -def start_osds(devices): - # Scan for ceph block devices - rescan_osd_devices() - if cmp_pkgrevno('ceph', "0.56.6") >= 0: - # Use ceph-disk activate for directory based OSD's - for dev_or_path in devices: - if os.path.exists(dev_or_path) and os.path.isdir(dev_or_path): - subprocess.check_call(['ceph-disk', 'activate', dev_or_path]) - - -def rescan_osd_devices(): - cmd = [ - 'udevadm', 'trigger', - '--subsystem-match=block', '--action=add' - ] - - subprocess.call(cmd) - - -_bootstrap_keyring = "/var/lib/ceph/bootstrap-osd/ceph.keyring" -_upgrade_keyring = "/var/lib/ceph/osd/ceph.client.osd-upgrade.keyring" - - -def is_bootstrapped(): - return os.path.exists(_bootstrap_keyring) - - -def wait_for_bootstrap(): - while not is_bootstrapped(): - time.sleep(3) - - -def import_osd_bootstrap_key(key): - if not os.path.exists(_bootstrap_keyring): - cmd = [ - "sudo", - "-u", - ceph_user(), - 'ceph-authtool', - _bootstrap_keyring, - '--create-keyring', - '--name=client.bootstrap-osd', - '--add-key={}'.format(key) - ] - subprocess.check_call(cmd) - - -def import_osd_upgrade_key(key): - if not os.path.exists(_upgrade_keyring): - cmd = [ - "sudo", - "-u", - ceph_user(), - 'ceph-authtool', - _upgrade_keyring, - '--create-keyring', - '--name=client.osd-upgrade', - '--add-key={}'.format(key) - ] - subprocess.check_call(cmd) - - -def generate_monitor_secret(): - cmd = [ - 'ceph-authtool', - '/dev/stdout', - '--name=mon.', - '--gen-key' - ] - res = str(subprocess.check_output(cmd).decode('UTF-8')) - - return "{}==".format(res.split('=')[1].strip()) - -# OSD caps taken from ceph-create-keys -_osd_bootstrap_caps = { - 'mon': [ - 'allow command osd create ...', - 'allow command osd crush set ...', - r'allow command auth add * osd allow\ * mon allow\ rwx', - 'allow command mon getmap' - ] -} - -_osd_bootstrap_caps_profile = { - 'mon': [ - 'allow profile bootstrap-osd' - ] -} - - -def parse_key(raw_key): - # get-or-create appears to have different output depending - # on whether its 'get' or 'create' - # 'create' just returns the key, 'get' is more verbose and - # needs parsing - key = None - if len(raw_key.splitlines()) == 1: - key = raw_key - else: - for element in raw_key.splitlines(): - if 'key' in element: - return element.split(' = ')[1].strip() # IGNORE:E1103 - return key - - -def get_osd_bootstrap_key(): - try: - # Attempt to get/create a key using the OSD bootstrap profile first - key = get_named_key('bootstrap-osd', - _osd_bootstrap_caps_profile) - except: - # If that fails try with the older style permissions - key = get_named_key('bootstrap-osd', - _osd_bootstrap_caps) - return key - - -_radosgw_keyring = "/etc/ceph/keyring.rados.gateway" - - -def import_radosgw_key(key): - if not os.path.exists(_radosgw_keyring): - cmd = [ - "sudo", - "-u", - ceph_user(), - 'ceph-authtool', - _radosgw_keyring, - '--create-keyring', - '--name=client.radosgw.gateway', - '--add-key={}'.format(key) - ] - subprocess.check_call(cmd) - -# OSD caps taken from ceph-create-keys -_radosgw_caps = { - 'mon': ['allow rw'], - 'osd': ['allow rwx'] -} -_upgrade_caps = { - 'mon': ['allow rwx'] -} - - -def get_radosgw_key(pool_list=None): - return get_named_key(name='radosgw.gateway', - caps=_radosgw_caps, - pool_list=pool_list) - - -def get_mds_key(name): - return create_named_keyring(entity='mds', - name=name, - caps=mds_caps) - - -_mds_bootstrap_caps_profile = { - 'mon': [ - 'allow profile bootstrap-mds' - ] -} - - -def get_mds_bootstrap_key(): - return get_named_key('bootstrap-mds', - _mds_bootstrap_caps_profile) - - -_default_caps = collections.OrderedDict([ - ('mon', ['allow r']), - ('osd', ['allow rwx']), -]) - -admin_caps = collections.OrderedDict([ - ('mds', ['allow *']), - ('mon', ['allow *']), - ('osd', ['allow *']) -]) - -mds_caps = collections.OrderedDict([ - ('osd', ['allow *']), - ('mds', ['allow']), - ('mon', ['allow rwx']), -]) - -osd_upgrade_caps = collections.OrderedDict([ - ('mon', ['allow command "config-key"', - 'allow command "osd tree"', - 'allow command "config-key list"', - 'allow command "config-key put"', - 'allow command "config-key get"', - 'allow command "config-key exists"', - 'allow command "osd out"', - 'allow command "osd in"', - 'allow command "osd rm"', - 'allow command "auth del"', - ]) -]) - - -def create_named_keyring(entity, name, caps=None): - caps = caps or _default_caps - cmd = [ - "sudo", - "-u", - ceph_user(), - 'ceph', - '--name', 'mon.', - '--keyring', - '/var/lib/ceph/mon/ceph-{}/keyring'.format( - socket.gethostname() - ), - 'auth', 'get-or-create', '{entity}.{name}'.format(entity=entity, - name=name), - ] - for subsystem, subcaps in caps.items(): - cmd.extend([subsystem, '; '.join(subcaps)]) - log("Calling check_output: {}".format(cmd), level=DEBUG) - return (parse_key(str(subprocess - .check_output(cmd) - .decode('UTF-8')) - .strip())) # IGNORE:E1103 - - -def get_upgrade_key(): - return get_named_key('upgrade-osd', _upgrade_caps) - - -def get_named_key(name, caps=None, pool_list=None): - """Retrieve a specific named cephx key. - - :param name: String Name of key to get. - :param pool_list: The list of pools to give access to - :param caps: dict of cephx capabilities - :returns: Returns a cephx key - """ - try: - # Does the key already exist? - output = str(subprocess.check_output( - [ - 'sudo', - '-u', ceph_user(), - 'ceph', - '--name', 'mon.', - '--keyring', - '/var/lib/ceph/mon/ceph-{}/keyring'.format( - socket.gethostname() - ), - 'auth', - 'get', - 'client.{}'.format(name), - ]).decode('UTF-8')).strip() - return parse_key(output) - except subprocess.CalledProcessError: - # Couldn't get the key, time to create it! - log("Creating new key for {}".format(name), level=DEBUG) - caps = caps or _default_caps - cmd = [ - "sudo", - "-u", - ceph_user(), - 'ceph', - '--name', 'mon.', - '--keyring', - '/var/lib/ceph/mon/ceph-{}/keyring'.format( - socket.gethostname() - ), - 'auth', 'get-or-create', 'client.{}'.format(name), - ] - # Add capabilities - for subsystem, subcaps in caps.items(): - if subsystem == 'osd': - if pool_list: - # This will output a string similar to: - # "pool=rgw pool=rbd pool=something" - pools = " ".join(['pool={0}'.format(i) for i in pool_list]) - subcaps[0] = subcaps[0] + " " + pools - cmd.extend([subsystem, '; '.join(subcaps)]) - - log("Calling check_output: {}".format(cmd), level=DEBUG) - return parse_key(str(subprocess - .check_output(cmd) - .decode('UTF-8')) - .strip()) # IGNORE:E1103 - - -def upgrade_key_caps(key, caps): - """ Upgrade key to have capabilities caps """ - if not is_leader(): - # Not the MON leader OR not clustered - return - cmd = [ - "sudo", "-u", ceph_user(), 'ceph', 'auth', 'caps', key - ] - for subsystem, subcaps in caps.items(): - cmd.extend([subsystem, '; '.join(subcaps)]) - subprocess.check_call(cmd) - - -@cached -def systemd(): - return CompareHostReleases(lsb_release()['DISTRIB_CODENAME']) >= 'vivid' - - -def bootstrap_monitor_cluster(secret): - hostname = socket.gethostname() - path = '/var/lib/ceph/mon/ceph-{}'.format(hostname) - done = '{}/done'.format(path) - if systemd(): - init_marker = '{}/systemd'.format(path) - else: - init_marker = '{}/upstart'.format(path) - - keyring = '/var/lib/ceph/tmp/{}.mon.keyring'.format(hostname) - - if os.path.exists(done): - log('bootstrap_monitor_cluster: mon already initialized.') - else: - # Ceph >= 0.61.3 needs this for ceph-mon fs creation - mkdir('/var/run/ceph', owner=ceph_user(), - group=ceph_user(), perms=0o755) - mkdir(path, owner=ceph_user(), group=ceph_user()) - # end changes for Ceph >= 0.61.3 - try: - add_keyring_to_ceph(keyring, - secret, - hostname, - path, - done, - init_marker) - - except: - raise - finally: - os.unlink(keyring) - - -@retry_on_exception(3, base_delay=5) -def add_keyring_to_ceph(keyring, secret, hostname, path, done, init_marker): - subprocess.check_call(['ceph-authtool', keyring, - '--create-keyring', '--name=mon.', - '--add-key={}'.format(secret), - '--cap', 'mon', 'allow *']) - subprocess.check_call(['ceph-mon', '--mkfs', - '-i', hostname, - '--keyring', keyring]) - chownr(path, ceph_user(), ceph_user()) - with open(done, 'w'): - pass - with open(init_marker, 'w'): - pass - - if systemd(): - subprocess.check_call(['systemctl', 'enable', 'ceph-mon']) - service_restart('ceph-mon') - else: - service_restart('ceph-mon-all') - - if cmp_pkgrevno('ceph', '12.0.0') >= 0: - # NOTE(jamespage): Later ceph releases require explicit - # call to ceph-create-keys to setup the - # admin keys for the cluster; this command - # will wait for quorum in the cluster before - # returning. - cmd = ['ceph-create-keys', '--id', hostname] - subprocess.check_call(cmd) - osstat = os.stat("/etc/ceph/ceph.client.admin.keyring") - if not osstat.st_size: - raise Exception - - -def update_monfs(): - hostname = socket.gethostname() - monfs = '/var/lib/ceph/mon/ceph-{}'.format(hostname) - if systemd(): - init_marker = '{}/systemd'.format(monfs) - else: - init_marker = '{}/upstart'.format(monfs) - if os.path.exists(monfs) and not os.path.exists(init_marker): - # Mark mon as managed by upstart so that - # it gets start correctly on reboots - with open(init_marker, 'w'): - pass - - -def maybe_zap_journal(journal_dev): - if is_osd_disk(journal_dev): - log('Looks like {} is already an OSD data' - ' or journal, skipping.'.format(journal_dev)) - return - zap_disk(journal_dev) - log("Zapped journal device {}".format(journal_dev)) - - -def get_partitions(dev): - cmd = ['partx', '--raw', '--noheadings', dev] - try: - out = str(subprocess.check_output(cmd).decode('UTF-8')).splitlines() - log("get partitions: {}".format(out), level=DEBUG) - return out - except subprocess.CalledProcessError as e: - log("Can't get info for {0}: {1}".format(dev, e.output)) - return [] - - -def find_least_used_utility_device(utility_devices): - """ - Find a utility device which has the smallest number of partitions - among other devices in the supplied list. - - :utility_devices: A list of devices to be used for filestore journal - or bluestore wal or db. - :return: string device name - """ - - usages = map(lambda a: (len(get_partitions(a)), a), utility_devices) - least = min(usages, key=lambda t: t[0]) - return least[1] - - -def get_devices(name): - """ Merge config and juju storage based devices - - :name: THe name of the device type, eg: wal, osd, journal - :returns: Set(device names), which are strings - """ - if config(name): - devices = [l.strip() for l in config(name).split(' ')] - else: - devices = [] - storage_ids = storage_list(name) - devices.extend((storage_get('location', s) for s in storage_ids)) - devices = filter(os.path.exists, devices) - - return set(devices) - - -def osdize(dev, osd_format, osd_journal, reformat_osd=False, - ignore_errors=False, encrypt=False, bluestore=False): - if dev.startswith('/dev'): - osdize_dev(dev, osd_format, osd_journal, - reformat_osd, ignore_errors, encrypt, - bluestore) - else: - osdize_dir(dev, encrypt, bluestore) - - -def osdize_dev(dev, osd_format, osd_journal, reformat_osd=False, - ignore_errors=False, encrypt=False, bluestore=False): - if not os.path.exists(dev): - log('Path {} does not exist - bailing'.format(dev)) - return - - if not is_block_device(dev): - log('Path {} is not a block device - bailing'.format(dev)) - return - - if is_osd_disk(dev) and not reformat_osd: - log('Looks like {} is already an' - ' OSD data or journal, skipping.'.format(dev)) - return - - if is_device_mounted(dev): - log('Looks like {} is in use, skipping.'.format(dev)) - return - - status_set('maintenance', 'Initializing device {}'.format(dev)) - cmd = ['ceph-disk', 'prepare'] - # Later versions of ceph support more options - if cmp_pkgrevno('ceph', '0.60') >= 0: - if encrypt: - cmd.append('--dmcrypt') - if cmp_pkgrevno('ceph', '0.48.3') >= 0: - if osd_format and not bluestore: - cmd.append('--fs-type') - cmd.append(osd_format) - - if reformat_osd: - cmd.append('--zap-disk') - - # NOTE(jamespage): enable experimental bluestore support - if cmp_pkgrevno('ceph', '10.2.0') >= 0 and bluestore: - cmd.append('--bluestore') - wal = get_devices('bluestore-wal') - if wal: - cmd.append('--block.wal') - least_used_wal = find_least_used_utility_device(wal) - cmd.append(least_used_wal) - db = get_devices('bluestore-db') - if db: - cmd.append('--block.db') - least_used_db = find_least_used_utility_device(db) - cmd.append(least_used_db) - elif cmp_pkgrevno('ceph', '12.1.0') >= 0 and not bluestore: - cmd.append('--filestore') - - cmd.append(dev) - - if osd_journal: - least_used = find_least_used_utility_device(osd_journal) - cmd.append(least_used) - else: - # Just provide the device - no other options - # for older versions of ceph - cmd.append(dev) - if reformat_osd: - zap_disk(dev) - - try: - log("osdize cmd: {}".format(cmd)) - subprocess.check_call(cmd) - except subprocess.CalledProcessError: - if ignore_errors: - log('Unable to initialize device: {}'.format(dev), WARNING) - else: - log('Unable to initialize device: {}'.format(dev), ERROR) - raise - - -def osdize_dir(path, encrypt=False, bluestore=False): - """Ask ceph-disk to prepare a directory to become an osd. - - :param path: str. The directory to osdize - :param encrypt: bool. Should the OSD directory be encrypted at rest - :returns: None - """ - if os.path.exists(os.path.join(path, 'upstart')): - log('Path {} is already configured as an OSD - bailing'.format(path)) - return - - if cmp_pkgrevno('ceph', "0.56.6") < 0: - log('Unable to use directories for OSDs with ceph < 0.56.6', - level=ERROR) - return - - mkdir(path, owner=ceph_user(), group=ceph_user(), perms=0o755) - chownr('/var/lib/ceph', ceph_user(), ceph_user()) - cmd = [ - 'sudo', '-u', ceph_user(), - 'ceph-disk', - 'prepare', - '--data-dir', - path - ] - if cmp_pkgrevno('ceph', '0.60') >= 0: - if encrypt: - cmd.append('--dmcrypt') - - # NOTE(icey): enable experimental bluestore support - if cmp_pkgrevno('ceph', '10.2.0') >= 0 and bluestore: - cmd.append('--bluestore') - elif cmp_pkgrevno('ceph', '12.1.0') >= 0 and not bluestore: - cmd.append('--filestore') - log("osdize dir cmd: {}".format(cmd)) - subprocess.check_call(cmd) - - -def filesystem_mounted(fs): - return subprocess.call(['grep', '-wqs', fs, '/proc/mounts']) == 0 - - -def get_running_osds(): - """Returns a list of the pids of the current running OSD daemons""" - cmd = ['pgrep', 'ceph-osd'] - try: - result = str(subprocess.check_output(cmd).decode('UTF-8')) - return result.split() - except subprocess.CalledProcessError: - return [] - - -def get_cephfs(service): - """List the Ceph Filesystems that exist. - - :param service: The service name to run the ceph command under - :returns: list. Returns a list of the ceph filesystems - """ - if get_version() < 0.86: - # This command wasn't introduced until 0.86 ceph - return [] - try: - output = str(subprocess - .check_output(["ceph", '--id', service, "fs", "ls"]) - .decode('UTF-8')) - if not output: - return [] - """ - Example subprocess output: - 'name: ip-172-31-23-165, metadata pool: ip-172-31-23-165_metadata, - data pools: [ip-172-31-23-165_data ]\n' - output: filesystems: ['ip-172-31-23-165'] - """ - filesystems = [] - for line in output.splitlines(): - parts = line.split(',') - for part in parts: - if "name" in part: - filesystems.append(part.split(' ')[1]) - except subprocess.CalledProcessError: - return [] - - -def wait_for_all_monitors_to_upgrade(new_version, upgrade_key): - """Fairly self explanatory name. This function will wait - for all monitors in the cluster to upgrade or it will - return after a timeout period has expired. - - :param new_version: str of the version to watch - :param upgrade_key: the cephx key name to use - """ - done = False - start_time = time.time() - monitor_list = [] - - mon_map = get_mon_map('admin') - if mon_map['monmap']['mons']: - for mon in mon_map['monmap']['mons']: - monitor_list.append(mon['name']) - while not done: - try: - done = all(monitor_key_exists(upgrade_key, "{}_{}_{}_done".format( - "mon", mon, new_version - )) for mon in monitor_list) - current_time = time.time() - if current_time > (start_time + 10 * 60): - raise Exception - else: - # Wait 30 seconds and test again if all monitors are upgraded - time.sleep(30) - except subprocess.CalledProcessError: - raise - - -# Edge cases: -# 1. Previous node dies on upgrade, can we retry? -def roll_monitor_cluster(new_version, upgrade_key): - """This is tricky to get right so here's what we're going to do. - - There's 2 possible cases: Either I'm first in line or not. - If I'm not first in line I'll wait a random time between 5-30 seconds - and test to see if the previous monitor is upgraded yet. - - :param new_version: str of the version to upgrade to - :param upgrade_key: the cephx key name to use when upgrading - """ - log('roll_monitor_cluster called with {}'.format(new_version)) - my_name = socket.gethostname() - monitor_list = [] - mon_map = get_mon_map('admin') - if mon_map['monmap']['mons']: - for mon in mon_map['monmap']['mons']: - monitor_list.append(mon['name']) - else: - status_set('blocked', 'Unable to get monitor cluster information') - sys.exit(1) - log('monitor_list: {}'.format(monitor_list)) - - # A sorted list of osd unit names - mon_sorted_list = sorted(monitor_list) - - try: - position = mon_sorted_list.index(my_name) - log("upgrade position: {}".format(position)) - if position == 0: - # I'm first! Roll - # First set a key to inform others I'm about to roll - lock_and_roll(upgrade_key=upgrade_key, - service='mon', - my_name=my_name, - version=new_version) - else: - # Check if the previous node has finished - status_set('waiting', - 'Waiting on {} to finish upgrading'.format( - mon_sorted_list[position - 1])) - wait_on_previous_node(upgrade_key=upgrade_key, - service='mon', - previous_node=mon_sorted_list[position - 1], - version=new_version) - lock_and_roll(upgrade_key=upgrade_key, - service='mon', - my_name=my_name, - version=new_version) - except ValueError: - log("Failed to find {} in list {}.".format( - my_name, mon_sorted_list)) - status_set('blocked', 'failed to upgrade monitor') - - -def upgrade_monitor(new_version): - """Upgrade the current ceph monitor to the new version - - :param new_version: String version to upgrade to. - """ - current_version = get_version() - status_set("maintenance", "Upgrading monitor") - log("Current ceph version is {}".format(current_version)) - log("Upgrading to: {}".format(new_version)) - - try: - add_source(config('source'), config('key')) - apt_update(fatal=True) - except subprocess.CalledProcessError as err: - log("Adding the ceph source failed with message: {}".format( - err)) - status_set("blocked", "Upgrade to {} failed".format(new_version)) - sys.exit(1) - try: - if systemd(): - for mon_id in get_local_mon_ids(): - service_stop('ceph-mon@{}'.format(mon_id)) - else: - service_stop('ceph-mon-all') - apt_install(packages=determine_packages(), fatal=True) - - # Ensure the files and directories under /var/lib/ceph is chowned - # properly as part of the move to the Jewel release, which moved the - # ceph daemons to running as ceph:ceph instead of root:root. - if new_version == 'jewel': - # Ensure the ownership of Ceph's directories is correct - owner = ceph_user() - chownr(path=os.path.join(os.sep, "var", "lib", "ceph"), - owner=owner, - group=owner, - follow_links=True) - - if systemd(): - for mon_id in get_local_mon_ids(): - service_start('ceph-mon@{}'.format(mon_id)) - else: - service_start('ceph-mon-all') - except subprocess.CalledProcessError as err: - log("Stopping ceph and upgrading packages failed " - "with message: {}".format(err)) - status_set("blocked", "Upgrade to {} failed".format(new_version)) - sys.exit(1) - - -def lock_and_roll(upgrade_key, service, my_name, version): - """Create a lock on the ceph monitor cluster and upgrade. - - :param upgrade_key: str. The cephx key to use - :param service: str. The cephx id to use - :param my_name: str. The current hostname - :param version: str. The version we are upgrading to - """ - start_timestamp = time.time() - - log('monitor_key_set {}_{}_{}_start {}'.format( - service, - my_name, - version, - start_timestamp)) - monitor_key_set(upgrade_key, "{}_{}_{}_start".format( - service, my_name, version), start_timestamp) - log("Rolling") - - # This should be quick - if service == 'osd': - upgrade_osd(version) - elif service == 'mon': - upgrade_monitor(version) - else: - log("Unknown service {}. Unable to upgrade".format(service), - level=ERROR) - log("Done") - - stop_timestamp = time.time() - # Set a key to inform others I am finished - log('monitor_key_set {}_{}_{}_done {}'.format(service, - my_name, - version, - stop_timestamp)) - status_set('maintenance', 'Finishing upgrade') - monitor_key_set(upgrade_key, "{}_{}_{}_done".format(service, - my_name, - version), - stop_timestamp) - - -def wait_on_previous_node(upgrade_key, service, previous_node, version): - """A lock that sleeps the current thread while waiting for the previous - node to finish upgrading. - - :param upgrade_key: - :param service: str. the cephx id to use - :param previous_node: str. The name of the previous node to wait on - :param version: str. The version we are upgrading to - :returns: None - """ - log("Previous node is: {}".format(previous_node)) - - previous_node_finished = monitor_key_exists( - upgrade_key, - "{}_{}_{}_done".format(service, previous_node, version)) - - while previous_node_finished is False: - log("{} is not finished. Waiting".format(previous_node)) - # Has this node been trying to upgrade for longer than - # 10 minutes? - # If so then move on and consider that node dead. - - # NOTE: This assumes the clusters clocks are somewhat accurate - # If the hosts clock is really far off it may cause it to skip - # the previous node even though it shouldn't. - current_timestamp = time.time() - previous_node_start_time = monitor_key_get( - upgrade_key, - "{}_{}_{}_start".format(service, previous_node, version)) - if (current_timestamp - (10 * 60)) > previous_node_start_time: - # Previous node is probably dead. Lets move on - if previous_node_start_time is not None: - log( - "Waited 10 mins on node {}. current time: {} > " - "previous node start time: {} Moving on".format( - previous_node, - (current_timestamp - (10 * 60)), - previous_node_start_time)) - return - else: - # I have to wait. Sleep a random amount of time and then - # check if I can lock,upgrade and roll. - wait_time = random.randrange(5, 30) - log('waiting for {} seconds'.format(wait_time)) - time.sleep(wait_time) - previous_node_finished = monitor_key_exists( - upgrade_key, - "{}_{}_{}_done".format(service, previous_node, version)) - - -def get_upgrade_position(osd_sorted_list, match_name): - """Return the upgrade position for the given osd. - - :param osd_sorted_list: list. Osds sorted - :param match_name: str. The osd name to match - :returns: int. The position or None if not found - """ - for index, item in enumerate(osd_sorted_list): - if item.name == match_name: - return index - return None - - -# Edge cases: -# 1. Previous node dies on upgrade, can we retry? -# 2. This assumes that the osd failure domain is not set to osd. -# It rolls an entire server at a time. -def roll_osd_cluster(new_version, upgrade_key): - """This is tricky to get right so here's what we're going to do. - - There's 2 possible cases: Either I'm first in line or not. - If I'm not first in line I'll wait a random time between 5-30 seconds - and test to see if the previous osd is upgraded yet. - - TODO: If you're not in the same failure domain it's safe to upgrade - 1. Examine all pools and adopt the most strict failure domain policy - Example: Pool 1: Failure domain = rack - Pool 2: Failure domain = host - Pool 3: Failure domain = row - - outcome: Failure domain = host - - :param new_version: str of the version to upgrade to - :param upgrade_key: the cephx key name to use when upgrading - """ - log('roll_osd_cluster called with {}'.format(new_version)) - my_name = socket.gethostname() - osd_tree = get_osd_tree(service=upgrade_key) - # A sorted list of osd unit names - osd_sorted_list = sorted(osd_tree) - log("osd_sorted_list: {}".format(osd_sorted_list)) - - try: - position = get_upgrade_position(osd_sorted_list, my_name) - log("upgrade position: {}".format(position)) - if position == 0: - # I'm first! Roll - # First set a key to inform others I'm about to roll - lock_and_roll(upgrade_key=upgrade_key, - service='osd', - my_name=my_name, - version=new_version) - else: - # Check if the previous node has finished - status_set('blocked', - 'Waiting on {} to finish upgrading'.format( - osd_sorted_list[position - 1].name)) - wait_on_previous_node( - upgrade_key=upgrade_key, - service='osd', - previous_node=osd_sorted_list[position - 1].name, - version=new_version) - lock_and_roll(upgrade_key=upgrade_key, - service='osd', - my_name=my_name, - version=new_version) - except ValueError: - log("Failed to find name {} in list {}".format( - my_name, osd_sorted_list)) - status_set('blocked', 'failed to upgrade osd') - - -def upgrade_osd(new_version): - """Upgrades the current osd - - :param new_version: str. The new version to upgrade to - """ - current_version = get_version() - status_set("maintenance", "Upgrading osd") - log("Current ceph version is {}".format(current_version)) - log("Upgrading to: {}".format(new_version)) - - try: - add_source(config('source'), config('key')) - apt_update(fatal=True) - except subprocess.CalledProcessError as err: - log("Adding the ceph sources failed with message: {}".format( - err)) - status_set("blocked", "Upgrade to {} failed".format(new_version)) - sys.exit(1) - - try: - # Upgrade the packages before restarting the daemons. - status_set('maintenance', 'Upgrading packages to %s' % new_version) - apt_install(packages=determine_packages(), fatal=True) - - # If the upgrade does not need an ownership update of any of the - # directories in the osd service directory, then simply restart - # all of the OSDs at the same time as this will be the fastest - # way to update the code on the node. - if not dirs_need_ownership_update('osd'): - log('Restarting all OSDs to load new binaries', DEBUG) - service_restart('ceph-osd-all') - return - - # Need to change the ownership of all directories which are not OSD - # directories as well. - # TODO - this should probably be moved to the general upgrade function - # and done before mon/osd. - update_owner(CEPH_BASE_DIR, recurse_dirs=False) - non_osd_dirs = filter(lambda x: not x == 'osd', - os.listdir(CEPH_BASE_DIR)) - non_osd_dirs = map(lambda x: os.path.join(CEPH_BASE_DIR, x), - non_osd_dirs) - for path in non_osd_dirs: - update_owner(path) - - # Fast service restart wasn't an option because each of the OSD - # directories need the ownership updated for all the files on - # the OSD. Walk through the OSDs one-by-one upgrading the OSD. - for osd_dir in _get_child_dirs(OSD_BASE_DIR): - try: - osd_num = _get_osd_num_from_dirname(osd_dir) - _upgrade_single_osd(osd_num, osd_dir) - except ValueError as ex: - # Directory could not be parsed - junk directory? - log('Could not parse osd directory %s: %s' % (osd_dir, ex), - WARNING) - continue - - except (subprocess.CalledProcessError, IOError) as err: - log("Stopping ceph and upgrading packages failed " - "with message: {}".format(err)) - status_set("blocked", "Upgrade to {} failed".format(new_version)) - sys.exit(1) - - -def _upgrade_single_osd(osd_num, osd_dir): - """Upgrades the single OSD directory. - - :param osd_num: the num of the OSD - :param osd_dir: the directory of the OSD to upgrade - :raises CalledProcessError: if an error occurs in a command issued as part - of the upgrade process - :raises IOError: if an error occurs reading/writing to a file as part - of the upgrade process - """ - stop_osd(osd_num) - disable_osd(osd_num) - update_owner(osd_dir) - enable_osd(osd_num) - start_osd(osd_num) - - -def stop_osd(osd_num): - """Stops the specified OSD number. - - :param osd_num: the osd number to stop - """ - if systemd(): - service_stop('ceph-osd@{}'.format(osd_num)) - else: - service_stop('ceph-osd', id=osd_num) - - -def start_osd(osd_num): - """Starts the specified OSD number. - - :param osd_num: the osd number to start. - """ - if systemd(): - service_start('ceph-osd@{}'.format(osd_num)) - else: - service_start('ceph-osd', id=osd_num) - - -def disable_osd(osd_num): - """Disables the specified OSD number. - - Ensures that the specified osd will not be automatically started at the - next reboot of the system. Due to differences between init systems, - this method cannot make any guarantees that the specified osd cannot be - started manually. - - :param osd_num: the osd id which should be disabled. - :raises CalledProcessError: if an error occurs invoking the systemd cmd - to disable the OSD - :raises IOError, OSError: if the attempt to read/remove the ready file in - an upstart enabled system fails - """ - if systemd(): - # When running under systemd, the individual ceph-osd daemons run as - # templated units and can be directly addressed by referring to the - # templated service name ceph-osd@. Additionally, systemd - # allows one to disable a specific templated unit by running the - # 'systemctl disable ceph-osd@' command. When disabled, the - # OSD should remain disabled until re-enabled via systemd. - # Note: disabling an already disabled service in systemd returns 0, so - # no need to check whether it is enabled or not. - cmd = ['systemctl', 'disable', 'ceph-osd@{}'.format(osd_num)] - subprocess.check_call(cmd) - else: - # Neither upstart nor the ceph-osd upstart script provides for - # disabling the starting of an OSD automatically. The specific OSD - # cannot be prevented from running manually, however it can be - # prevented from running automatically on reboot by removing the - # 'ready' file in the OSD's root directory. This is due to the - # ceph-osd-all upstart script checking for the presence of this file - # before starting the OSD. - ready_file = os.path.join(OSD_BASE_DIR, 'ceph-{}'.format(osd_num), - 'ready') - if os.path.exists(ready_file): - os.unlink(ready_file) - - -def enable_osd(osd_num): - """Enables the specified OSD number. - - Ensures that the specified osd_num will be enabled and ready to start - automatically in the event of a reboot. - - :param osd_num: the osd id which should be enabled. - :raises CalledProcessError: if the call to the systemd command issued - fails when enabling the service - :raises IOError: if the attempt to write the ready file in an usptart - enabled system fails - """ - if systemd(): - cmd = ['systemctl', 'enable', 'ceph-osd@{}'.format(osd_num)] - subprocess.check_call(cmd) - else: - # When running on upstart, the OSDs are started via the ceph-osd-all - # upstart script which will only start the osd if it has a 'ready' - # file. Make sure that file exists. - ready_file = os.path.join(OSD_BASE_DIR, 'ceph-{}'.format(osd_num), - 'ready') - with open(ready_file, 'w') as f: - f.write('ready') - - # Make sure the correct user owns the file. It shouldn't be necessary - # as the upstart script should run with root privileges, but its better - # to have all the files matching ownership. - update_owner(ready_file) - - -def update_owner(path, recurse_dirs=True): - """Changes the ownership of the specified path. - - Changes the ownership of the specified path to the new ceph daemon user - using the system's native chown functionality. This may take awhile, - so this method will issue a set_status for any changes of ownership which - recurses into directory structures. - - :param path: the path to recursively change ownership for - :param recurse_dirs: boolean indicating whether to recursively change the - ownership of all the files in a path's subtree or to - simply change the ownership of the path. - :raises CalledProcessError: if an error occurs issuing the chown system - command - """ - user = ceph_user() - user_group = '{ceph_user}:{ceph_user}'.format(ceph_user=user) - cmd = ['chown', user_group, path] - if os.path.isdir(path) and recurse_dirs: - status_set('maintenance', ('Updating ownership of %s to %s' % - (path, user))) - cmd.insert(1, '-R') - - log('Changing ownership of {path} to {user}'.format( - path=path, user=user_group), DEBUG) - start = datetime.now() - subprocess.check_call(cmd) - elapsed_time = (datetime.now() - start) - - log('Took {secs} seconds to change the ownership of path: {path}'.format( - secs=elapsed_time.total_seconds(), path=path), DEBUG) - - -def list_pools(service): - """This will list the current pools that Ceph has - - :param service: String service id to run under - :returns: list. Returns a list of the ceph pools. - :raises: CalledProcessError if the subprocess fails to run. - """ - try: - pool_list = [] - pools = str(subprocess - .check_output(['rados', '--id', service, 'lspools']) - .decode('UTF-8')) - for pool in pools.splitlines(): - pool_list.append(pool) - return pool_list - except subprocess.CalledProcessError as err: - log("rados lspools failed with error: {}".format(err.output)) - raise - - -def dirs_need_ownership_update(service): - """Determines if directories still need change of ownership. - - Examines the set of directories under the /var/lib/ceph/{service} directory - and determines if they have the correct ownership or not. This is - necessary due to the upgrade from Hammer to Jewel where the daemon user - changes from root: to ceph:. - - :param service: the name of the service folder to check (e.g. osd, mon) - :returns: boolean. True if the directories need a change of ownership, - False otherwise. - :raises IOError: if an error occurs reading the file stats from one of - the child directories. - :raises OSError: if the specified path does not exist or some other error - """ - expected_owner = expected_group = ceph_user() - path = os.path.join(CEPH_BASE_DIR, service) - for child in _get_child_dirs(path): - curr_owner, curr_group = owner(child) - - if (curr_owner == expected_owner) and (curr_group == expected_group): - continue - - log('Directory "%s" needs its ownership updated' % child, DEBUG) - return True - - # All child directories had the expected ownership - return False - -# A dict of valid ceph upgrade paths. Mapping is old -> new -UPGRADE_PATHS = { - 'firefly': 'hammer', - 'hammer': 'jewel', - 'jewel': 'luminous', -} - -# Map UCA codenames to ceph codenames -UCA_CODENAME_MAP = { - 'icehouse': 'firefly', - 'juno': 'firefly', - 'kilo': 'hammer', - 'liberty': 'hammer', - 'mitaka': 'jewel', - 'newton': 'jewel', - 'ocata': 'jewel', - 'pike': 'luminous', - 'queens': 'luminous', -} - - -def pretty_print_upgrade_paths(): - """Pretty print supported upgrade paths for ceph""" - return ["{} -> {}".format(key, value) - for key, value in UPGRADE_PATHS.items()] - - -def resolve_ceph_version(source): - """Resolves a version of ceph based on source configuration - based on Ubuntu Cloud Archive pockets. - - @param: source: source configuration option of charm - :returns: ceph release codename or None if not resolvable - """ - os_release = get_os_codename_install_source(source) - return UCA_CODENAME_MAP.get(os_release) - - -def get_ceph_pg_stat(): - """Returns the result of ceph pg stat. - - :returns: dict - """ - try: - tree = str(subprocess - .check_output(['ceph', 'pg', 'stat', '--format=json']) - .decode('UTF-8')) - try: - json_tree = json.loads(tree) - if not json_tree['num_pg_by_state']: - return None - return json_tree - except ValueError as v: - log("Unable to parse ceph pg stat json: {}. Error: {}".format( - tree, v)) - raise - except subprocess.CalledProcessError as e: - log("ceph pg stat command failed with message: {}".format(e)) - raise - - -def get_ceph_health(): - """Returns the health of the cluster from a 'ceph status' - - :returns: dict tree of ceph status - :raises: CalledProcessError if our ceph command fails to get the overall - status, use get_ceph_health()['overall_status']. - """ - try: - tree = str(subprocess - .check_output(['ceph', 'status', '--format=json']) - .decode('UTF-8')) - try: - json_tree = json.loads(tree) - # Make sure children are present in the json - if not json_tree['overall_status']: - return None - - return json_tree - except ValueError as v: - log("Unable to parse ceph tree json: {}. Error: {}".format( - tree, v)) - raise - except subprocess.CalledProcessError as e: - log("ceph status command failed with message: {}".format(e)) - raise - - -def reweight_osd(osd_num, new_weight): - """Changes the crush weight of an OSD to the value specified. - - :param osd_num: the osd id which should be changed - :param new_weight: the new weight for the OSD - :returns: bool. True if output looks right, else false. - :raises CalledProcessError: if an error occurs invoking the systemd cmd - """ - try: - cmd_result = str(subprocess - .check_output(['ceph', 'osd', 'crush', - 'reweight', "osd.{}".format(osd_num), - new_weight], - stderr=subprocess.STDOUT) - .decode('UTF-8')) - expected_result = "reweighted item id {ID} name \'osd.{ID}\'".format( - ID=osd_num) + " to {}".format(new_weight) - log(cmd_result) - if expected_result in cmd_result: - return True - return False - except subprocess.CalledProcessError as e: - log("ceph osd crush reweight command failed" - " with message: {}".format(e)) - raise - - -def determine_packages(): - """Determines packages for installation. - - :returns: list of ceph packages - """ - if is_container(): - PACKAGES.remove('ntp') - - return PACKAGES - - -def bootstrap_manager(): - hostname = socket.gethostname() - path = '/var/lib/ceph/mgr/ceph-{}'.format(hostname) - keyring = os.path.join(path, 'keyring') - - if os.path.exists(keyring): - log('bootstrap_manager: mgr already initialized.') - else: - mkdir(path, owner=ceph_user(), group=ceph_user()) - subprocess.check_call(['ceph', 'auth', 'get-or-create', - 'mgr.{}'.format(hostname), 'mon', - 'allow profile mgr', 'osd', 'allow *', - 'mds', 'allow *', '--out-file', - keyring]) - chownr(path, ceph_user(), ceph_user()) - - unit = 'ceph-mgr@{}'.format(hostname) - subprocess.check_call(['systemctl', 'enable', unit]) - service_restart(unit) - - -def osd_noout(enable): - """Sets or unsets 'noout' - - :param enable: bool. True to set noout, False to unset. - :returns: bool. True if output looks right. - :raises CalledProcessError: if an error occurs invoking the systemd cmd - """ - operation = { - True: 'set', - False: 'unset', - } - try: - subprocess.check_call(['ceph', '--id', 'admin', - 'osd', operation[enable], - 'noout']) - log('running ceph osd {} noout'.format(operation[enable])) - return True - except subprocess.CalledProcessError as e: - log(e) - raise diff --git a/metadata.yaml b/metadata.yaml deleted file mode 100644 index a76ae96..0000000 --- a/metadata.yaml +++ /dev/null @@ -1,44 +0,0 @@ -name: ceph -summary: Highly scalable distributed storage -maintainer: OpenStack Charmers -description: | - Ceph is a distributed storage and network file system designed to provide - excellent performance, reliability, and scalability. -tags: - - openstack - - storage - - file-servers - - misc -series: - - xenial - - artful - - zesty - - trusty -peers: - mon: - interface: ceph -extra-bindings: - cluster: - public: -provides: - nrpe-external-master: - interface: nrpe-external-master - scope: container - client: - interface: ceph-client - osd: - interface: ceph-osd - radosgw: - interface: ceph-radosgw - bootstrap-source: - interface: ceph-bootstrap -storage: - osd-devices: - type: block - multiple: - range: 0- - minimum-size: 1G - osd-journal: - type: block - multiple: - range: 0-1 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index db0af4d..0000000 --- a/requirements.txt +++ /dev/null @@ -1,13 +0,0 @@ -# The order of packages is significant, because pip processes them in the order -# of appearance. Changing the order has an impact on the overall integration -# process, which may cause wedges in the gate later. -pbr>=1.8.0,<1.9.0 -PyYAML>=3.1.0 -simplejson>=2.2.0 -netifaces>=0.10.4 -netaddr>=0.7.12,!=0.7.16 -Jinja2>=2.6 # BSD License (3 clause) -six>=1.9.0 -dnspython>=1.12.0 -psutil>=1.1.1,<2.0.0 -pyudev diff --git a/revision b/revision deleted file mode 100644 index ffda4e7..0000000 --- a/revision +++ /dev/null @@ -1 +0,0 @@ -105 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 37083b6..0000000 --- a/setup.cfg +++ /dev/null @@ -1,5 +0,0 @@ -[nosetests] -verbosity=2 -with-coverage=1 -cover-erase=1 -cover-package=hooks diff --git a/templates/ceph.conf b/templates/ceph.conf deleted file mode 100644 index b90e3b4..0000000 --- a/templates/ceph.conf +++ /dev/null @@ -1,90 +0,0 @@ -[global] -{%- if old_auth %} -auth supported = {{ auth_supported }} -{%- else %} -auth cluster required = {{ auth_supported }} -auth service required = {{ auth_supported }} -auth client required = {{ auth_supported }} -{%- endif %} - -mon host = {{ mon_hosts }} -fsid = {{ fsid }} - -log to syslog = {{ use_syslog }} -err to syslog = {{ use_syslog }} -clog to syslog = {{ use_syslog }} -mon cluster log to syslog = {{ use_syslog }} -debug mon = {{ loglevel }}/5 -debug osd = {{ loglevel }}/5 - -{% if ceph_public_network is string %} -public network = {{ ceph_public_network }} -{%- endif %} -{%- if ceph_cluster_network is string %} -cluster network = {{ ceph_cluster_network }} -{%- endif %} -{%- if public_addr %} -public addr = {{ public_addr }} -{%- endif %} -{%- if cluster_addr %} -cluster addr = {{ cluster_addr }} -{%- endif %} -{% if global -%} -# The following are user-provided options provided via the config-flags charm option. -# User-provided [global] section config -{% for key in global -%} -{{ key }} = {{ global[key] }} -{% endfor %} -{% endif %} - -{% if bluestore_experimental and bluestore -%} -enable experimental unrecoverable data corrupting features = bluestore rocksdb -{%- endif %} - -{% if rbd_features %} -rbd default features = {{ rbd_features }} -{% endif %} - -[mon] -keyring = /var/lib/ceph/mon/$cluster-$id/keyring -{% if mon -%} -# The following are user-provided options provided via the config-flags charm option. -# User-provided [mon] section config -{% for key in mon -%} -{{ key }} = {{ mon[key] }} -{% endfor %} -{% endif %} - -[mds] -keyring = /var/lib/ceph/mds/$cluster-$id/keyring -{% if mds -%} -# The following are user-provided options provided via the config-flags charm option. -# User-provided [mds] section config -{% for key in mds -%} -{{ key }} = {{ mds[key] }} -{% endfor %} -{% endif %} - - -[osd] -keyring = /var/lib/ceph/osd/$cluster-$id/keyring -{% if bluestore -%} -{% if not bluestore_experimental -%} -osd objectstore = bluestore -{%- endif -%} -{%- else %} -osd journal size = {{ osd_journal_size }} -filestore xattr use omap = true -journal dio = {{ dio }} -{%- endif %} - -{%- if short_object_len %} -osd max object name len = 256 -osd max object namespace len = 64 -{% endif %} -{% if osd -%} -# The following are user-provided options provided via the config-flags charm option. -{% for key in osd -%} -{{ key }} = {{ osd[key] }} -{% endfor %} -{% endif %} diff --git a/test-requirements.txt b/test-requirements.txt deleted file mode 100644 index 9edd4bb..0000000 --- a/test-requirements.txt +++ /dev/null @@ -1,27 +0,0 @@ -# The order of packages is significant, because pip processes them in the order -# of appearance. Changing the order has an impact on the overall integration -# process, which may cause wedges in the gate later. -coverage>=3.6 -mock>=1.2 -flake8>=2.2.4,<=2.4.1 -os-testr>=0.4.1 -charm-tools>=2.0.0 -requests==2.6.0 -# BEGIN: Amulet OpenStack Charm Helper Requirements -# Liberty client lower constraints -amulet>=1.14.3,<2.0 -bundletester>=0.6.1,<1.0 -python-ceilometerclient>=1.5.0 -python-cinderclient>=1.4.0 -python-glanceclient>=1.1.0 -python-heatclient>=0.8.0 -python-keystoneclient>=1.7.1 -python-neutronclient>=3.1.0 -python-novaclient>=2.30.1 -python-openstackclient>=1.7.0 -python-swiftclient>=2.6.0 -pika>=0.10.0,<1.0 -distro-info -# END: Amulet OpenStack Charm Helper Requirements -# NOTE: workaround for 14.04 pip/tox -pytz diff --git a/tests/README.md b/tests/README.md deleted file mode 100644 index 046be7f..0000000 --- a/tests/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# Overview - -This directory provides Amulet tests to verify basic deployment functionality -from the perspective of this charm, its requirements and its features, as -exercised in a subset of the full OpenStack deployment test bundle topology. - -For full details on functional testing of OpenStack charms please refer to -the [functional testing](http://docs.openstack.org/developer/charm-guide/testing.html#functional-testing) -section of the OpenStack Charm Guide. diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py deleted file mode 100644 index 8a5dffa..0000000 --- a/tests/basic_deployment.py +++ /dev/null @@ -1,662 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import amulet -import time - -from charmhelpers.contrib.openstack.amulet.deployment import ( - OpenStackAmuletDeployment -) -from charmhelpers.contrib.openstack.amulet.utils import ( # noqa - OpenStackAmuletUtils, - DEBUG, - # ERROR -) - -# Use DEBUG to turn on debug logging -u = OpenStackAmuletUtils(DEBUG) - - -class CephBasicDeployment(OpenStackAmuletDeployment): - """Amulet tests on a basic ceph deployment.""" - - def __init__(self, series=None, openstack=None, source=None, stable=False): - """Deploy the entire test environment.""" - super(CephBasicDeployment, self).__init__(series, openstack, source, - stable) - self._add_services() - self._add_relations() - self._configure_services() - self._deploy() - - u.log.info('Waiting on extended status checks...') - exclude_services = [] - - # Wait for deployment ready msgs, except exclusions - self._auto_wait_for_status(exclude_services=exclude_services) - - self.d.sentry.wait() - self._initialize_tests() - - def _add_services(self): - """Add services - - Add the services that we're testing, where ceph is local, - and the rest of the service are from lp branches that are - compatible with the local charm (e.g. stable or next). - """ - this_service = {'name': 'ceph', 'units': 3} - other_services = [ - {'name': 'percona-cluster'}, - {'name': 'keystone'}, - {'name': 'rabbitmq-server'}, - {'name': 'nova-compute'}, - {'name': 'glance'}, - {'name': 'cinder'}, - {'name': 'cinder-ceph'}, - {'name': 'ceph-osd'}, - ] - super(CephBasicDeployment, self)._add_services(this_service, - other_services) - - def _add_relations(self): - """Add all of the relations for the services.""" - relations = { - 'nova-compute:shared-db': 'percona-cluster:shared-db', - 'nova-compute:amqp': 'rabbitmq-server:amqp', - 'nova-compute:image-service': 'glance:image-service', - 'nova-compute:ceph': 'ceph:client', - 'keystone:shared-db': 'percona-cluster:shared-db', - 'glance:shared-db': 'percona-cluster:shared-db', - 'glance:identity-service': 'keystone:identity-service', - 'glance:amqp': 'rabbitmq-server:amqp', - 'glance:ceph': 'ceph:client', - 'cinder:shared-db': 'percona-cluster:shared-db', - 'cinder:identity-service': 'keystone:identity-service', - 'cinder:amqp': 'rabbitmq-server:amqp', - 'cinder:image-service': 'glance:image-service', - 'cinder-ceph:storage-backend': 'cinder:storage-backend', - 'cinder-ceph:ceph': 'ceph:client', - 'ceph-osd:mon': 'ceph:osd' - } - super(CephBasicDeployment, self)._add_relations(relations) - - def _configure_services(self): - """Configure all of the services.""" - keystone_config = {'admin-password': 'openstack', - 'admin-token': 'ubuntutesting'} - pxc_config = { - 'innodb-buffer-pool-size': '256M', - 'max-connections': 1000, - 'root-password': 'ChangeMe123', - 'sst-password': 'ChangeMe123', - } - cinder_config = {'block-device': 'None', 'glance-api-version': '2'} - - # Include a non-existent device as osd-devices is a whitelist, - # and this will catch cases where proposals attempt to change that. - ceph_config = { - 'monitor-count': '3', - 'auth-supported': 'none', - 'fsid': '6547bd3e-1397-11e2-82e5-53567c8d32dc', - 'monitor-secret': 'AQCXrnZQwI7KGBAAiPofmKEXKxu5bUzoYLVkbQ==', - 'osd-reformat': 'yes', - 'ephemeral-unmount': '/mnt', - 'osd-devices': '/dev/vdb /srv/ceph /dev/test-non-existent' - } - - ceph_osd_config = { - 'osd-reformat': 'yes', - 'ephemeral-unmount': '/mnt', - 'osd-devices': '/dev/vdb /srv/ceph /dev/test-non-existent' - } - - configs = {'keystone': keystone_config, - 'percona-cluster': pxc_config, - 'cinder': cinder_config, - 'ceph': ceph_config, - 'ceph-osd': ceph_osd_config} - super(CephBasicDeployment, self)._configure_services(configs) - - def _initialize_tests(self): - """Perform final initialization before tests get run.""" - # Access the sentries for inspecting service units - self.pxc_sentry = self.d.sentry['percona-cluster'][0] - self.keystone_sentry = self.d.sentry['keystone'][0] - self.rabbitmq_sentry = self.d.sentry['rabbitmq-server'][0] - self.nova_sentry = self.d.sentry['nova-compute'][0] - self.glance_sentry = self.d.sentry['glance'][0] - self.cinder_sentry = self.d.sentry['cinder'][0] - self.cinder_ceph_sentry = self.d.sentry['cinder-ceph'][0] - self.ceph0_sentry = self.d.sentry['ceph'][0] - self.ceph1_sentry = self.d.sentry['ceph'][1] - self.ceph2_sentry = self.d.sentry['ceph'][2] - self.ceph_osd_sentry = self.d.sentry['ceph-osd'][0] - u.log.debug('openstack release val: {}'.format( - self._get_openstack_release())) - u.log.debug('openstack release str: {}'.format( - self._get_openstack_release_string())) - - # Authenticate admin with keystone - self.keystone = u.authenticate_keystone_admin(self.keystone_sentry, - user='admin', - password='openstack', - tenant='admin') - # Authenticate admin with cinder endpoint - self.cinder = u.authenticate_cinder_admin(self.keystone_sentry, - username='admin', - password='openstack', - tenant='admin') - # Authenticate admin with glance endpoint - self.glance = u.authenticate_glance_admin(self.keystone) - - # Authenticate admin with nova endpoint - self.nova = u.authenticate_nova_user(self.keystone, - user='admin', - password='openstack', - tenant='admin') - - # Create a demo tenant/role/user - self.demo_tenant = 'demoTenant' - self.demo_role = 'demoRole' - self.demo_user = 'demoUser' - if not u.tenant_exists(self.keystone, self.demo_tenant): - tenant = self.keystone.tenants.create(tenant_name=self.demo_tenant, - description='demo tenant', - enabled=True) - self.keystone.roles.create(name=self.demo_role) - self.keystone.users.create(name=self.demo_user, - password='password', - tenant_id=tenant.id, - email='demo@demo.com') - - # Authenticate demo user with keystone - self.keystone_demo = u.authenticate_keystone_user(self.keystone, - self.demo_user, - 'password', - self.demo_tenant) - - # Authenticate demo user with nova-api - self.nova_demo = u.authenticate_nova_user(self.keystone, - self.demo_user, - 'password', - self.demo_tenant) - - def test_100_ceph_processes(self): - """Verify that the expected service processes are running - on each ceph unit.""" - - # Process name and quantity of processes to expect on each unit - ceph_processes = { - 'ceph-mon': 1, - 'ceph-osd': 2 - } - - # Units with process names and PID quantities expected - expected_processes = { - self.ceph0_sentry: ceph_processes, - self.ceph1_sentry: ceph_processes, - self.ceph2_sentry: ceph_processes - } - - actual_pids = u.get_unit_process_ids(expected_processes) - ret = u.validate_unit_process_ids(expected_processes, actual_pids) - if ret: - amulet.raise_status(amulet.FAIL, msg=ret) - - def test_102_services(self): - """Verify the expected services are running on the service units.""" - - services = { - self.rabbitmq_sentry: ['rabbitmq-server'], - self.nova_sentry: ['nova-compute'], - self.keystone_sentry: ['keystone'], - self.glance_sentry: ['glance-registry', - 'glance-api'], - self.cinder_sentry: ['cinder-scheduler', - 'cinder-volume'], - } - - if self._get_openstack_release() < self.xenial_ocata: - services[self.cinder_sentry].append('cinder-api') - - if self._get_openstack_release() < self.xenial_mitaka: - # For upstart systems only. Ceph services under systemd - # are checked by process name instead. - ceph_services = [ - 'ceph-mon-all', - 'ceph-mon id=`hostname`', - 'ceph-osd-all', - 'ceph-osd id={}'.format(u.get_ceph_osd_id_cmd(0)), - 'ceph-osd id={}'.format(u.get_ceph_osd_id_cmd(1)) - ] - services[self.ceph0_sentry] = ceph_services - services[self.ceph1_sentry] = ceph_services - services[self.ceph2_sentry] = ceph_services - - if self._get_openstack_release() >= self.trusty_liberty: - services[self.keystone_sentry] = ['apache2'] - - ret = u.validate_services_by_name(services) - if ret: - amulet.raise_status(amulet.FAIL, msg=ret) - - def test_200_ceph_nova_client_relation(self): - """Verify the ceph to nova ceph-client relation data.""" - u.log.debug('Checking ceph:nova-compute ceph relation data...') - unit = self.ceph0_sentry - relation = ['client', 'nova-compute:ceph'] - expected = { - 'private-address': u.valid_ip, - 'auth': 'none', - 'key': u.not_null - } - - ret = u.validate_relation_data(unit, relation, expected) - if ret: - message = u.relation_error('ceph to nova ceph-client', ret) - amulet.raise_status(amulet.FAIL, msg=message) - - def test_201_nova_ceph_client_relation(self): - """Verify the nova to ceph client relation data.""" - u.log.debug('Checking nova-compute:ceph ceph-client relation data...') - unit = self.nova_sentry - relation = ['ceph', 'ceph:client'] - expected = { - 'private-address': u.valid_ip - } - - ret = u.validate_relation_data(unit, relation, expected) - if ret: - message = u.relation_error('nova to ceph ceph-client', ret) - amulet.raise_status(amulet.FAIL, msg=message) - - def test_202_ceph_glance_client_relation(self): - """Verify the ceph to glance ceph-client relation data.""" - u.log.debug('Checking ceph:glance client relation data...') - unit = self.ceph1_sentry - relation = ['client', 'glance:ceph'] - expected = { - 'private-address': u.valid_ip, - 'auth': 'none', - 'key': u.not_null - } - - ret = u.validate_relation_data(unit, relation, expected) - if ret: - message = u.relation_error('ceph to glance ceph-client', ret) - amulet.raise_status(amulet.FAIL, msg=message) - - def test_203_glance_ceph_client_relation(self): - """Verify the glance to ceph client relation data.""" - u.log.debug('Checking glance:ceph client relation data...') - unit = self.glance_sentry - relation = ['ceph', 'ceph:client'] - expected = { - 'private-address': u.valid_ip - } - - ret = u.validate_relation_data(unit, relation, expected) - if ret: - message = u.relation_error('glance to ceph ceph-client', ret) - amulet.raise_status(amulet.FAIL, msg=message) - - def test_204_ceph_cinder_client_relation(self): - """Verify the ceph to cinder ceph-client relation data.""" - u.log.debug('Checking ceph:cinder ceph relation data...') - unit = self.ceph2_sentry - relation = ['client', 'cinder-ceph:ceph'] - expected = { - 'private-address': u.valid_ip, - 'auth': 'none', - 'key': u.not_null - } - - ret = u.validate_relation_data(unit, relation, expected) - if ret: - message = u.relation_error('ceph to cinder ceph-client', ret) - amulet.raise_status(amulet.FAIL, msg=message) - - def test_205_cinder_ceph_client_relation(self): - """Verify the cinder to ceph ceph-client relation data.""" - u.log.debug('Checking cinder:ceph ceph relation data...') - unit = self.cinder_ceph_sentry - relation = ['ceph', 'ceph:client'] - expected = { - 'private-address': u.valid_ip - } - - ret = u.validate_relation_data(unit, relation, expected) - if ret: - message = u.relation_error('cinder to ceph ceph-client', ret) - amulet.raise_status(amulet.FAIL, msg=message) - - def test_300_ceph_config(self): - """Verify the data in the ceph config file.""" - u.log.debug('Checking ceph config file data...') - unit = self.ceph0_sentry - conf = '/etc/ceph/ceph.conf' - expected = { - 'global': { - 'fsid': '6547bd3e-1397-11e2-82e5-53567c8d32dc', - 'log to syslog': 'false', - 'err to syslog': 'false', - 'clog to syslog': 'false', - 'mon cluster log to syslog': 'false', - 'auth cluster required': 'none', - 'auth service required': 'none', - 'auth client required': 'none' - }, - 'mon': { - 'keyring': '/var/lib/ceph/mon/$cluster-$id/keyring' - }, - 'mds': { - 'keyring': '/var/lib/ceph/mds/$cluster-$id/keyring' - }, - 'osd': { - 'keyring': '/var/lib/ceph/osd/$cluster-$id/keyring', - 'osd journal size': '1024', - 'filestore xattr use omap': 'true' - }, - } - - for section, pairs in expected.iteritems(): - ret = u.validate_config_data(unit, conf, section, pairs) - if ret: - message = "ceph config error: {}".format(ret) - amulet.raise_status(amulet.FAIL, msg=message) - - def test_302_cinder_rbd_config(self): - """Verify the cinder config file data regarding ceph.""" - u.log.debug('Checking cinder (rbd) config file data...') - unit = self.cinder_sentry - conf = '/etc/cinder/cinder.conf' - section_key = 'cinder-ceph' - expected = { - section_key: { - 'volume_driver': 'cinder.volume.drivers.rbd.RBDDriver' - } - } - - for section, pairs in expected.iteritems(): - ret = u.validate_config_data(unit, conf, section, pairs) - if ret: - message = "cinder (rbd) config error: {}".format(ret) - amulet.raise_status(amulet.FAIL, msg=message) - - def test_304_glance_rbd_config(self): - """Verify the glance config file data regarding ceph.""" - u.log.debug('Checking glance (rbd) config file data...') - unit = self.glance_sentry - conf = '/etc/glance/glance-api.conf' - config = { - 'default_store': 'rbd', - 'rbd_store_ceph_conf': '/etc/ceph/ceph.conf', - 'rbd_store_user': 'glance', - 'rbd_store_pool': 'glance', - 'rbd_store_chunk_size': '8' - } - - if self._get_openstack_release() >= self.trusty_kilo: - # Kilo or later - config['stores'] = ('glance.store.filesystem.Store,' - 'glance.store.http.Store,' - 'glance.store.rbd.Store') - section = 'glance_store' - else: - # Juno or earlier - section = 'DEFAULT' - - expected = {section: config} - for section, pairs in expected.iteritems(): - ret = u.validate_config_data(unit, conf, section, pairs) - if ret: - message = "glance (rbd) config error: {}".format(ret) - amulet.raise_status(amulet.FAIL, msg=message) - - def test_306_nova_rbd_config(self): - """Verify the nova config file data regarding ceph.""" - u.log.debug('Checking nova (rbd) config file data...') - unit = self.nova_sentry - conf = '/etc/nova/nova.conf' - expected = { - 'libvirt': { - 'rbd_user': 'nova-compute', - 'rbd_secret_uuid': u.not_null - } - } - for section, pairs in expected.iteritems(): - ret = u.validate_config_data(unit, conf, section, pairs) - if ret: - message = "nova (rbd) config error: {}".format(ret) - amulet.raise_status(amulet.FAIL, msg=message) - - def test_400_ceph_check_osd_pools(self): - """Check osd pools on all ceph units, expect them to be - identical, and expect specific pools to be present.""" - u.log.debug('Checking pools on ceph units...') - - expected_pools = self.get_ceph_expected_pools() - results = [] - sentries = [ - self.ceph0_sentry, - self.ceph1_sentry, - self.ceph2_sentry - ] - - # Check for presence of expected pools on each unit - u.log.debug('Expected pools: {}'.format(expected_pools)) - for sentry_unit in sentries: - pools = u.get_ceph_pools(sentry_unit) - results.append(pools) - - for expected_pool in expected_pools: - if expected_pool not in pools: - msg = ('{} does not have pool: ' - '{}'.format(sentry_unit.info['unit_name'], - expected_pool)) - amulet.raise_status(amulet.FAIL, msg=msg) - u.log.debug('{} has (at least) the expected ' - 'pools.'.format(sentry_unit.info['unit_name'])) - - # Check that all units returned the same pool name:id data - ret = u.validate_list_of_identical_dicts(results) - if ret: - u.log.debug('Pool list results: {}'.format(results)) - msg = ('{}; Pool list results are not identical on all ' - 'ceph units.'.format(ret)) - amulet.raise_status(amulet.FAIL, msg=msg) - else: - u.log.debug('Pool list on all ceph units produced the ' - 'same results (OK).') - - def test_402_pause_resume_actions(self): - """Veryfy that pause/resume works""" - u.log.debug("Testing pause") - cmd = "ceph -s" - - sentry_unit = self.ceph0_sentry - action_id = u.run_action(sentry_unit, 'pause-health') - assert u.wait_on_action(action_id), "Pause health action failed." - - output, code = sentry_unit.run(cmd) - if 'nodown' not in output or 'noout' not in output: - amulet.raise_status(amulet.FAIL, msg="Missing noout,nodown") - - u.log.debug("Testing resume") - action_id = u.run_action(sentry_unit, 'resume-health') - assert u.wait_on_action(action_id), "Resume health action failed." - - output, code = sentry_unit.run(cmd) - if 'nodown' in output or 'noout' in output: - amulet.raise_status(amulet.FAIL, msg="Still has noout,nodown") - - def test_410_ceph_cinder_vol_create(self): - """Create and confirm a ceph-backed cinder volume, and inspect - ceph cinder pool object count as the volume is created - and deleted.""" - sentry_unit = self.ceph0_sentry - obj_count_samples = [] - pool_size_samples = [] - pools = u.get_ceph_pools(self.ceph0_sentry) - cinder_pool = pools['cinder-ceph'] - - # Check ceph cinder pool object count, disk space usage and pool name - u.log.debug('Checking ceph cinder pool original samples...') - pool_name, obj_count, kb_used = u.get_ceph_pool_sample(sentry_unit, - cinder_pool) - obj_count_samples.append(obj_count) - pool_size_samples.append(kb_used) - - expected = 'cinder-ceph' - if pool_name != expected: - msg = ('Ceph pool {} unexpected name (actual, expected): ' - '{}. {}'.format(cinder_pool, pool_name, expected)) - amulet.raise_status(amulet.FAIL, msg=msg) - - # Create ceph-backed cinder volume - cinder_vol = u.create_cinder_volume(self.cinder) - - # Re-check ceph cinder pool object count and disk usage - time.sleep(10) - u.log.debug('Checking ceph cinder pool samples after volume create...') - pool_name, obj_count, kb_used = u.get_ceph_pool_sample(sentry_unit, - cinder_pool) - obj_count_samples.append(obj_count) - pool_size_samples.append(kb_used) - - # Delete ceph-backed cinder volume - u.delete_resource(self.cinder.volumes, cinder_vol, msg="cinder volume") - - # Final check, ceph cinder pool object count and disk usage - time.sleep(10) - u.log.debug('Checking ceph cinder pool after volume delete...') - pool_name, obj_count, kb_used = u.get_ceph_pool_sample(sentry_unit, - cinder_pool) - obj_count_samples.append(obj_count) - pool_size_samples.append(kb_used) - - # Validate ceph cinder pool object count samples over time - ret = u.validate_ceph_pool_samples(obj_count_samples, - "cinder pool object count") - if ret: - amulet.raise_status(amulet.FAIL, msg=ret) - - # Luminous (pike) ceph seems more efficient at disk usage so we cannot - # grantee the ordering of kb_used - if self._get_openstack_release() < self.xenial_pike: - # Validate ceph cinder pool disk space usage samples over time - ret = u.validate_ceph_pool_samples(pool_size_samples, - "cinder pool disk usage") - if ret: - amulet.raise_status(amulet.FAIL, msg=ret) - - def test_412_ceph_glance_image_create_delete(self): - """Create and confirm a ceph-backed glance image, and inspect - ceph glance pool object count as the image is created - and deleted.""" - sentry_unit = self.ceph0_sentry - obj_count_samples = [] - pool_size_samples = [] - pools = u.get_ceph_pools(self.ceph0_sentry) - glance_pool = pools['glance'] - - # Check ceph glance pool object count, disk space usage and pool name - u.log.debug('Checking ceph glance pool original samples...') - pool_name, obj_count, kb_used = u.get_ceph_pool_sample(sentry_unit, - glance_pool) - obj_count_samples.append(obj_count) - pool_size_samples.append(kb_used) - - expected = 'glance' - if pool_name != expected: - msg = ('Ceph glance pool {} unexpected name (actual, ' - 'expected): {}. {}'.format(glance_pool, - pool_name, expected)) - amulet.raise_status(amulet.FAIL, msg=msg) - - # Create ceph-backed glance image - glance_img = u.create_cirros_image(self.glance, "cirros-image-1") - - # Re-check ceph glance pool object count and disk usage - time.sleep(10) - u.log.debug('Checking ceph glance pool samples after image create...') - pool_name, obj_count, kb_used = u.get_ceph_pool_sample(sentry_unit, - glance_pool) - obj_count_samples.append(obj_count) - pool_size_samples.append(kb_used) - - # Delete ceph-backed glance image - u.delete_resource(self.glance.images, - glance_img, msg="glance image") - - # Final check, ceph glance pool object count and disk usage - time.sleep(10) - u.log.debug('Checking ceph glance pool samples after image delete...') - pool_name, obj_count, kb_used = u.get_ceph_pool_sample(sentry_unit, - glance_pool) - obj_count_samples.append(obj_count) - pool_size_samples.append(kb_used) - - # Validate ceph glance pool object count samples over time - ret = u.validate_ceph_pool_samples(obj_count_samples, - "glance pool object count") - if ret: - amulet.raise_status(amulet.FAIL, msg=ret) - - # Validate ceph glance pool disk space usage samples over time - ret = u.validate_ceph_pool_samples(pool_size_samples, - "glance pool disk usage") - if ret: - amulet.raise_status(amulet.FAIL, msg=ret) - - def test_499_ceph_cmds_exit_zero(self): - """Check basic functionality of ceph cli commands against - all ceph units.""" - sentry_units = [ - self.ceph0_sentry, - self.ceph1_sentry, - self.ceph2_sentry - ] - commands = [ - 'sudo ceph health', - 'sudo ceph mds stat', - 'sudo ceph pg stat', - 'sudo ceph osd stat', - 'sudo ceph mon stat', - ] - ret = u.check_commands_on_units(commands, sentry_units) - if ret: - amulet.raise_status(amulet.FAIL, msg=ret) - - # FYI: No restart check as ceph services do not restart - # when charm config changes, unless monitor count increases. - - def test_910_pause_and_resume(self): - """The services can be paused and resumed. """ - u.log.debug('Checking pause and resume actions...') - sentry_unit = self.ceph0_sentry - - assert u.status_get(sentry_unit)[0] == "active" - - action_id = u.run_action(sentry_unit, "pause") - assert u.wait_on_action(action_id), "Pause action failed." - assert u.status_get(sentry_unit)[0] == "maintenance" - - action_id = u.run_action(sentry_unit, "resume") - assert u.wait_on_action(action_id), "Resume action failed." - assert u.status_get(sentry_unit)[0] == "active" - u.log.debug('OK') diff --git a/tests/charmhelpers/__init__.py b/tests/charmhelpers/__init__.py deleted file mode 100644 index e7aa471..0000000 --- a/tests/charmhelpers/__init__.py +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Bootstrap charm-helpers, installing its dependencies if necessary using -# only standard libraries. -from __future__ import print_function -from __future__ import absolute_import - -import functools -import inspect -import subprocess -import sys - -try: - import six # flake8: noqa -except ImportError: - if sys.version_info.major == 2: - subprocess.check_call(['apt-get', 'install', '-y', 'python-six']) - else: - subprocess.check_call(['apt-get', 'install', '-y', 'python3-six']) - import six # flake8: noqa - -try: - import yaml # flake8: noqa -except ImportError: - if sys.version_info.major == 2: - subprocess.check_call(['apt-get', 'install', '-y', 'python-yaml']) - else: - subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml']) - import yaml # flake8: noqa - - -# Holds a list of mapping of mangled function names that have been deprecated -# using the @deprecate decorator below. This is so that the warning is only -# printed once for each usage of the function. -__deprecated_functions = {} - - -def deprecate(warning, date=None, log=None): - """Add a deprecation warning the first time the function is used. - The date, which is a string in semi-ISO8660 format indicate the year-month - that the function is officially going to be removed. - - usage: - - @deprecate('use core/fetch/add_source() instead', '2017-04') - def contributed_add_source_thing(...): - ... - - And it then prints to the log ONCE that the function is deprecated. - The reason for passing the logging function (log) is so that hookenv.log - can be used for a charm if needed. - - :param warning: String to indicat where it has moved ot. - :param date: optional sting, in YYYY-MM format to indicate when the - function will definitely (probably) be removed. - :param log: The log function to call to log. If not, logs to stdout - """ - def wrap(f): - - @functools.wraps(f) - def wrapped_f(*args, **kwargs): - try: - module = inspect.getmodule(f) - file = inspect.getsourcefile(f) - lines = inspect.getsourcelines(f) - f_name = "{}-{}-{}..{}-{}".format( - module.__name__, file, lines[0], lines[-1], f.__name__) - except (IOError, TypeError): - # assume it was local, so just use the name of the function - f_name = f.__name__ - if f_name not in __deprecated_functions: - __deprecated_functions[f_name] = True - s = "DEPRECATION WARNING: Function {} is being removed".format( - f.__name__) - if date: - s = "{} on/around {}".format(s, date) - if warning: - s = "{} : {}".format(s, warning) - if log: - log(s) - else: - print(s) - return f(*args, **kwargs) - return wrapped_f - return wrap diff --git a/tests/charmhelpers/contrib/__init__.py b/tests/charmhelpers/contrib/__init__.py deleted file mode 100644 index d7567b8..0000000 --- a/tests/charmhelpers/contrib/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tests/charmhelpers/contrib/amulet/__init__.py b/tests/charmhelpers/contrib/amulet/__init__.py deleted file mode 100644 index d7567b8..0000000 --- a/tests/charmhelpers/contrib/amulet/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tests/charmhelpers/contrib/amulet/deployment.py b/tests/charmhelpers/contrib/amulet/deployment.py deleted file mode 100644 index 9c65518..0000000 --- a/tests/charmhelpers/contrib/amulet/deployment.py +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import amulet -import os -import six - - -class AmuletDeployment(object): - """Amulet deployment. - - This class provides generic Amulet deployment and test runner - methods. - """ - - def __init__(self, series=None): - """Initialize the deployment environment.""" - self.series = None - - if series: - self.series = series - self.d = amulet.Deployment(series=self.series) - else: - self.d = amulet.Deployment() - - def _add_services(self, this_service, other_services): - """Add services. - - Add services to the deployment where this_service is the local charm - that we're testing and other_services are the other services that - are being used in the local amulet tests. - """ - if this_service['name'] != os.path.basename(os.getcwd()): - s = this_service['name'] - msg = "The charm's root directory name needs to be {}".format(s) - amulet.raise_status(amulet.FAIL, msg=msg) - - if 'units' not in this_service: - this_service['units'] = 1 - - self.d.add(this_service['name'], units=this_service['units'], - constraints=this_service.get('constraints')) - - for svc in other_services: - if 'location' in svc: - branch_location = svc['location'] - elif self.series: - branch_location = 'cs:{}/{}'.format(self.series, svc['name']), - else: - branch_location = None - - if 'units' not in svc: - svc['units'] = 1 - - self.d.add(svc['name'], charm=branch_location, units=svc['units'], - constraints=svc.get('constraints')) - - def _add_relations(self, relations): - """Add all of the relations for the services.""" - for k, v in six.iteritems(relations): - self.d.relate(k, v) - - def _configure_services(self, configs): - """Configure all of the services.""" - for service, config in six.iteritems(configs): - self.d.configure(service, config) - - def _deploy(self): - """Deploy environment and wait for all hooks to finish executing.""" - timeout = int(os.environ.get('AMULET_SETUP_TIMEOUT', 900)) - try: - self.d.setup(timeout=timeout) - self.d.sentry.wait(timeout=timeout) - except amulet.helpers.TimeoutError: - amulet.raise_status( - amulet.FAIL, - msg="Deployment timed out ({}s)".format(timeout) - ) - except Exception: - raise - - def run_tests(self): - """Run all of the methods that are prefixed with 'test_'.""" - for test in dir(self): - if test.startswith('test_'): - getattr(self, test)() diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py deleted file mode 100644 index 8a6b764..0000000 --- a/tests/charmhelpers/contrib/amulet/utils.py +++ /dev/null @@ -1,821 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import io -import json -import logging -import os -import re -import socket -import subprocess -import sys -import time -import uuid - -import amulet -import distro_info -import six -from six.moves import configparser -if six.PY3: - from urllib import parse as urlparse -else: - import urlparse - - -class AmuletUtils(object): - """Amulet utilities. - - This class provides common utility functions that are used by Amulet - tests. - """ - - def __init__(self, log_level=logging.ERROR): - self.log = self.get_logger(level=log_level) - self.ubuntu_releases = self.get_ubuntu_releases() - - def get_logger(self, name="amulet-logger", level=logging.DEBUG): - """Get a logger object that will log to stdout.""" - log = logging - logger = log.getLogger(name) - fmt = log.Formatter("%(asctime)s %(funcName)s " - "%(levelname)s: %(message)s") - - handler = log.StreamHandler(stream=sys.stdout) - handler.setLevel(level) - handler.setFormatter(fmt) - - logger.addHandler(handler) - logger.setLevel(level) - - return logger - - def valid_ip(self, ip): - if re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", ip): - return True - else: - return False - - def valid_url(self, url): - p = re.compile( - r'^(?:http|ftp)s?://' - r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # noqa - r'localhost|' - r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' - r'(?::\d+)?' - r'(?:/?|[/?]\S+)$', - re.IGNORECASE) - if p.match(url): - return True - else: - return False - - def get_ubuntu_release_from_sentry(self, sentry_unit): - """Get Ubuntu release codename from sentry unit. - - :param sentry_unit: amulet sentry/service unit pointer - :returns: list of strings - release codename, failure message - """ - msg = None - cmd = 'lsb_release -cs' - release, code = sentry_unit.run(cmd) - if code == 0: - self.log.debug('{} lsb_release: {}'.format( - sentry_unit.info['unit_name'], release)) - else: - msg = ('{} `{}` returned {} ' - '{}'.format(sentry_unit.info['unit_name'], - cmd, release, code)) - if release not in self.ubuntu_releases: - msg = ("Release ({}) not found in Ubuntu releases " - "({})".format(release, self.ubuntu_releases)) - return release, msg - - def validate_services(self, commands): - """Validate that lists of commands succeed on service units. Can be - used to verify system services are running on the corresponding - service units. - - :param commands: dict with sentry keys and arbitrary command list vals - :returns: None if successful, Failure string message otherwise - """ - self.log.debug('Checking status of system services...') - - # /!\ DEPRECATION WARNING (beisner): - # New and existing tests should be rewritten to use - # validate_services_by_name() as it is aware of init systems. - self.log.warn('DEPRECATION WARNING: use ' - 'validate_services_by_name instead of validate_services ' - 'due to init system differences.') - - for k, v in six.iteritems(commands): - for cmd in v: - output, code = k.run(cmd) - self.log.debug('{} `{}` returned ' - '{}'.format(k.info['unit_name'], - cmd, code)) - if code != 0: - return "command `{}` returned {}".format(cmd, str(code)) - return None - - def validate_services_by_name(self, sentry_services): - """Validate system service status by service name, automatically - detecting init system based on Ubuntu release codename. - - :param sentry_services: dict with sentry keys and svc list values - :returns: None if successful, Failure string message otherwise - """ - self.log.debug('Checking status of system services...') - - # Point at which systemd became a thing - systemd_switch = self.ubuntu_releases.index('vivid') - - for sentry_unit, services_list in six.iteritems(sentry_services): - # Get lsb_release codename from unit - release, ret = self.get_ubuntu_release_from_sentry(sentry_unit) - if ret: - return ret - - for service_name in services_list: - if (self.ubuntu_releases.index(release) >= systemd_switch or - service_name in ['rabbitmq-server', 'apache2', - 'memcached']): - # init is systemd (or regular sysv) - cmd = 'sudo service {} status'.format(service_name) - output, code = sentry_unit.run(cmd) - service_running = code == 0 - elif self.ubuntu_releases.index(release) < systemd_switch: - # init is upstart - cmd = 'sudo status {}'.format(service_name) - output, code = sentry_unit.run(cmd) - service_running = code == 0 and "start/running" in output - - self.log.debug('{} `{}` returned ' - '{}'.format(sentry_unit.info['unit_name'], - cmd, code)) - if not service_running: - return u"command `{}` returned {} {}".format( - cmd, output, str(code)) - return None - - def _get_config(self, unit, filename): - """Get a ConfigParser object for parsing a unit's config file.""" - file_contents = unit.file_contents(filename) - - # NOTE(beisner): by default, ConfigParser does not handle options - # with no value, such as the flags used in the mysql my.cnf file. - # https://bugs.python.org/issue7005 - config = configparser.ConfigParser(allow_no_value=True) - config.readfp(io.StringIO(file_contents)) - return config - - def validate_config_data(self, sentry_unit, config_file, section, - expected): - """Validate config file data. - - Verify that the specified section of the config file contains - the expected option key:value pairs. - - Compare expected dictionary data vs actual dictionary data. - The values in the 'expected' dictionary can be strings, bools, ints, - longs, or can be a function that evaluates a variable and returns a - bool. - """ - self.log.debug('Validating config file data ({} in {} on {})' - '...'.format(section, config_file, - sentry_unit.info['unit_name'])) - config = self._get_config(sentry_unit, config_file) - - if section != 'DEFAULT' and not config.has_section(section): - return "section [{}] does not exist".format(section) - - for k in expected.keys(): - if not config.has_option(section, k): - return "section [{}] is missing option {}".format(section, k) - - actual = config.get(section, k) - v = expected[k] - if (isinstance(v, six.string_types) or - isinstance(v, bool) or - isinstance(v, six.integer_types)): - # handle explicit values - if actual != v: - return "section [{}] {}:{} != expected {}:{}".format( - section, k, actual, k, expected[k]) - # handle function pointers, such as not_null or valid_ip - elif not v(actual): - return "section [{}] {}:{} != expected {}:{}".format( - section, k, actual, k, expected[k]) - return None - - def _validate_dict_data(self, expected, actual): - """Validate dictionary data. - - Compare expected dictionary data vs actual dictionary data. - The values in the 'expected' dictionary can be strings, bools, ints, - longs, or can be a function that evaluates a variable and returns a - bool. - """ - self.log.debug('actual: {}'.format(repr(actual))) - self.log.debug('expected: {}'.format(repr(expected))) - - for k, v in six.iteritems(expected): - if k in actual: - if (isinstance(v, six.string_types) or - isinstance(v, bool) or - isinstance(v, six.integer_types)): - # handle explicit values - if v != actual[k]: - return "{}:{}".format(k, actual[k]) - # handle function pointers, such as not_null or valid_ip - elif not v(actual[k]): - return "{}:{}".format(k, actual[k]) - else: - return "key '{}' does not exist".format(k) - return None - - def validate_relation_data(self, sentry_unit, relation, expected): - """Validate actual relation data based on expected relation data.""" - actual = sentry_unit.relation(relation[0], relation[1]) - return self._validate_dict_data(expected, actual) - - def _validate_list_data(self, expected, actual): - """Compare expected list vs actual list data.""" - for e in expected: - if e not in actual: - return "expected item {} not found in actual list".format(e) - return None - - def not_null(self, string): - if string is not None: - return True - else: - return False - - def _get_file_mtime(self, sentry_unit, filename): - """Get last modification time of file.""" - return sentry_unit.file_stat(filename)['mtime'] - - def _get_dir_mtime(self, sentry_unit, directory): - """Get last modification time of directory.""" - return sentry_unit.directory_stat(directory)['mtime'] - - def _get_proc_start_time(self, sentry_unit, service, pgrep_full=None): - """Get start time of a process based on the last modification time - of the /proc/pid directory. - - :sentry_unit: The sentry unit to check for the service on - :service: service name to look for in process table - :pgrep_full: [Deprecated] Use full command line search mode with pgrep - :returns: epoch time of service process start - :param commands: list of bash commands - :param sentry_units: list of sentry unit pointers - :returns: None if successful; Failure message otherwise - """ - if pgrep_full is not None: - # /!\ DEPRECATION WARNING (beisner): - # No longer implemented, as pidof is now used instead of pgrep. - # https://bugs.launchpad.net/charm-helpers/+bug/1474030 - self.log.warn('DEPRECATION WARNING: pgrep_full bool is no ' - 'longer implemented re: lp 1474030.') - - pid_list = self.get_process_id_list(sentry_unit, service) - pid = pid_list[0] - proc_dir = '/proc/{}'.format(pid) - self.log.debug('Pid for {} on {}: {}'.format( - service, sentry_unit.info['unit_name'], pid)) - - return self._get_dir_mtime(sentry_unit, proc_dir) - - def service_restarted(self, sentry_unit, service, filename, - pgrep_full=None, sleep_time=20): - """Check if service was restarted. - - Compare a service's start time vs a file's last modification time - (such as a config file for that service) to determine if the service - has been restarted. - """ - # /!\ DEPRECATION WARNING (beisner): - # This method is prone to races in that no before-time is known. - # Use validate_service_config_changed instead. - - # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now - # used instead of pgrep. pgrep_full is still passed through to ensure - # deprecation WARNS. lp1474030 - self.log.warn('DEPRECATION WARNING: use ' - 'validate_service_config_changed instead of ' - 'service_restarted due to known races.') - - time.sleep(sleep_time) - if (self._get_proc_start_time(sentry_unit, service, pgrep_full) >= - self._get_file_mtime(sentry_unit, filename)): - return True - else: - return False - - def service_restarted_since(self, sentry_unit, mtime, service, - pgrep_full=None, sleep_time=20, - retry_count=30, retry_sleep_time=10): - """Check if service was been started after a given time. - - Args: - sentry_unit (sentry): The sentry unit to check for the service on - mtime (float): The epoch time to check against - service (string): service name to look for in process table - pgrep_full: [Deprecated] Use full command line search mode with pgrep - sleep_time (int): Initial sleep time (s) before looking for file - retry_sleep_time (int): Time (s) to sleep between retries - retry_count (int): If file is not found, how many times to retry - - Returns: - bool: True if service found and its start time it newer than mtime, - False if service is older than mtime or if service was - not found. - """ - # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now - # used instead of pgrep. pgrep_full is still passed through to ensure - # deprecation WARNS. lp1474030 - - unit_name = sentry_unit.info['unit_name'] - self.log.debug('Checking that %s service restarted since %s on ' - '%s' % (service, mtime, unit_name)) - time.sleep(sleep_time) - proc_start_time = None - tries = 0 - while tries <= retry_count and not proc_start_time: - try: - proc_start_time = self._get_proc_start_time(sentry_unit, - service, - pgrep_full) - self.log.debug('Attempt {} to get {} proc start time on {} ' - 'OK'.format(tries, service, unit_name)) - except IOError as e: - # NOTE(beisner) - race avoidance, proc may not exist yet. - # https://bugs.launchpad.net/charm-helpers/+bug/1474030 - self.log.debug('Attempt {} to get {} proc start time on {} ' - 'failed\n{}'.format(tries, service, - unit_name, e)) - time.sleep(retry_sleep_time) - tries += 1 - - if not proc_start_time: - self.log.warn('No proc start time found, assuming service did ' - 'not start') - return False - if proc_start_time >= mtime: - self.log.debug('Proc start time is newer than provided mtime' - '(%s >= %s) on %s (OK)' % (proc_start_time, - mtime, unit_name)) - return True - else: - self.log.warn('Proc start time (%s) is older than provided mtime ' - '(%s) on %s, service did not ' - 'restart' % (proc_start_time, mtime, unit_name)) - return False - - def config_updated_since(self, sentry_unit, filename, mtime, - sleep_time=20, retry_count=30, - retry_sleep_time=10): - """Check if file was modified after a given time. - - Args: - sentry_unit (sentry): The sentry unit to check the file mtime on - filename (string): The file to check mtime of - mtime (float): The epoch time to check against - sleep_time (int): Initial sleep time (s) before looking for file - retry_sleep_time (int): Time (s) to sleep between retries - retry_count (int): If file is not found, how many times to retry - - Returns: - bool: True if file was modified more recently than mtime, False if - file was modified before mtime, or if file not found. - """ - unit_name = sentry_unit.info['unit_name'] - self.log.debug('Checking that %s updated since %s on ' - '%s' % (filename, mtime, unit_name)) - time.sleep(sleep_time) - file_mtime = None - tries = 0 - while tries <= retry_count and not file_mtime: - try: - file_mtime = self._get_file_mtime(sentry_unit, filename) - self.log.debug('Attempt {} to get {} file mtime on {} ' - 'OK'.format(tries, filename, unit_name)) - except IOError as e: - # NOTE(beisner) - race avoidance, file may not exist yet. - # https://bugs.launchpad.net/charm-helpers/+bug/1474030 - self.log.debug('Attempt {} to get {} file mtime on {} ' - 'failed\n{}'.format(tries, filename, - unit_name, e)) - time.sleep(retry_sleep_time) - tries += 1 - - if not file_mtime: - self.log.warn('Could not determine file mtime, assuming ' - 'file does not exist') - return False - - if file_mtime >= mtime: - self.log.debug('File mtime is newer than provided mtime ' - '(%s >= %s) on %s (OK)' % (file_mtime, - mtime, unit_name)) - return True - else: - self.log.warn('File mtime is older than provided mtime' - '(%s < on %s) on %s' % (file_mtime, - mtime, unit_name)) - return False - - def validate_service_config_changed(self, sentry_unit, mtime, service, - filename, pgrep_full=None, - sleep_time=20, retry_count=30, - retry_sleep_time=10): - """Check service and file were updated after mtime - - Args: - sentry_unit (sentry): The sentry unit to check for the service on - mtime (float): The epoch time to check against - service (string): service name to look for in process table - filename (string): The file to check mtime of - pgrep_full: [Deprecated] Use full command line search mode with pgrep - sleep_time (int): Initial sleep in seconds to pass to test helpers - retry_count (int): If service is not found, how many times to retry - retry_sleep_time (int): Time in seconds to wait between retries - - Typical Usage: - u = OpenStackAmuletUtils(ERROR) - ... - mtime = u.get_sentry_time(self.cinder_sentry) - self.d.configure('cinder', {'verbose': 'True', 'debug': 'True'}) - if not u.validate_service_config_changed(self.cinder_sentry, - mtime, - 'cinder-api', - '/etc/cinder/cinder.conf') - amulet.raise_status(amulet.FAIL, msg='update failed') - Returns: - bool: True if both service and file where updated/restarted after - mtime, False if service is older than mtime or if service was - not found or if filename was modified before mtime. - """ - - # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now - # used instead of pgrep. pgrep_full is still passed through to ensure - # deprecation WARNS. lp1474030 - - service_restart = self.service_restarted_since( - sentry_unit, mtime, - service, - pgrep_full=pgrep_full, - sleep_time=sleep_time, - retry_count=retry_count, - retry_sleep_time=retry_sleep_time) - - config_update = self.config_updated_since( - sentry_unit, - filename, - mtime, - sleep_time=sleep_time, - retry_count=retry_count, - retry_sleep_time=retry_sleep_time) - - return service_restart and config_update - - def get_sentry_time(self, sentry_unit): - """Return current epoch time on a sentry""" - cmd = "date +'%s'" - return float(sentry_unit.run(cmd)[0]) - - def relation_error(self, name, data): - return 'unexpected relation data in {} - {}'.format(name, data) - - def endpoint_error(self, name, data): - return 'unexpected endpoint data in {} - {}'.format(name, data) - - def get_ubuntu_releases(self): - """Return a list of all Ubuntu releases in order of release.""" - _d = distro_info.UbuntuDistroInfo() - _release_list = _d.all - return _release_list - - def file_to_url(self, file_rel_path): - """Convert a relative file path to a file URL.""" - _abs_path = os.path.abspath(file_rel_path) - return urlparse.urlparse(_abs_path, scheme='file').geturl() - - def check_commands_on_units(self, commands, sentry_units): - """Check that all commands in a list exit zero on all - sentry units in a list. - - :param commands: list of bash commands - :param sentry_units: list of sentry unit pointers - :returns: None if successful; Failure message otherwise - """ - self.log.debug('Checking exit codes for {} commands on {} ' - 'sentry units...'.format(len(commands), - len(sentry_units))) - for sentry_unit in sentry_units: - for cmd in commands: - output, code = sentry_unit.run(cmd) - if code == 0: - self.log.debug('{} `{}` returned {} ' - '(OK)'.format(sentry_unit.info['unit_name'], - cmd, code)) - else: - return ('{} `{}` returned {} ' - '{}'.format(sentry_unit.info['unit_name'], - cmd, code, output)) - return None - - def get_process_id_list(self, sentry_unit, process_name, - expect_success=True): - """Get a list of process ID(s) from a single sentry juju unit - for a single process name. - - :param sentry_unit: Amulet sentry instance (juju unit) - :param process_name: Process name - :param expect_success: If False, expect the PID to be missing, - raise if it is present. - :returns: List of process IDs - """ - cmd = 'pidof -x "{}"'.format(process_name) - if not expect_success: - cmd += " || exit 0 && exit 1" - output, code = sentry_unit.run(cmd) - if code != 0: - msg = ('{} `{}` returned {} ' - '{}'.format(sentry_unit.info['unit_name'], - cmd, code, output)) - amulet.raise_status(amulet.FAIL, msg=msg) - return str(output).split() - - def get_unit_process_ids(self, unit_processes, expect_success=True): - """Construct a dict containing unit sentries, process names, and - process IDs. - - :param unit_processes: A dictionary of Amulet sentry instance - to list of process names. - :param expect_success: if False expect the processes to not be - running, raise if they are. - :returns: Dictionary of Amulet sentry instance to dictionary - of process names to PIDs. - """ - pid_dict = {} - for sentry_unit, process_list in six.iteritems(unit_processes): - pid_dict[sentry_unit] = {} - for process in process_list: - pids = self.get_process_id_list( - sentry_unit, process, expect_success=expect_success) - pid_dict[sentry_unit].update({process: pids}) - return pid_dict - - def validate_unit_process_ids(self, expected, actual): - """Validate process id quantities for services on units.""" - self.log.debug('Checking units for running processes...') - self.log.debug('Expected PIDs: {}'.format(expected)) - self.log.debug('Actual PIDs: {}'.format(actual)) - - if len(actual) != len(expected): - return ('Unit count mismatch. expected, actual: {}, ' - '{} '.format(len(expected), len(actual))) - - for (e_sentry, e_proc_names) in six.iteritems(expected): - e_sentry_name = e_sentry.info['unit_name'] - if e_sentry in actual.keys(): - a_proc_names = actual[e_sentry] - else: - return ('Expected sentry ({}) not found in actual dict data.' - '{}'.format(e_sentry_name, e_sentry)) - - if len(e_proc_names.keys()) != len(a_proc_names.keys()): - return ('Process name count mismatch. expected, actual: {}, ' - '{}'.format(len(expected), len(actual))) - - for (e_proc_name, e_pids), (a_proc_name, a_pids) in \ - zip(e_proc_names.items(), a_proc_names.items()): - if e_proc_name != a_proc_name: - return ('Process name mismatch. expected, actual: {}, ' - '{}'.format(e_proc_name, a_proc_name)) - - a_pids_length = len(a_pids) - fail_msg = ('PID count mismatch. {} ({}) expected, actual: ' - '{}, {} ({})'.format(e_sentry_name, e_proc_name, - e_pids, a_pids_length, - a_pids)) - - # If expected is a list, ensure at least one PID quantity match - if isinstance(e_pids, list) and \ - a_pids_length not in e_pids: - return fail_msg - # If expected is not bool and not list, - # ensure PID quantities match - elif not isinstance(e_pids, bool) and \ - not isinstance(e_pids, list) and \ - a_pids_length != e_pids: - return fail_msg - # If expected is bool True, ensure 1 or more PIDs exist - elif isinstance(e_pids, bool) and \ - e_pids is True and a_pids_length < 1: - return fail_msg - # If expected is bool False, ensure 0 PIDs exist - elif isinstance(e_pids, bool) and \ - e_pids is False and a_pids_length != 0: - return fail_msg - else: - self.log.debug('PID check OK: {} {} {}: ' - '{}'.format(e_sentry_name, e_proc_name, - e_pids, a_pids)) - return None - - def validate_list_of_identical_dicts(self, list_of_dicts): - """Check that all dicts within a list are identical.""" - hashes = [] - for _dict in list_of_dicts: - hashes.append(hash(frozenset(_dict.items()))) - - self.log.debug('Hashes: {}'.format(hashes)) - if len(set(hashes)) == 1: - self.log.debug('Dicts within list are identical') - else: - return 'Dicts within list are not identical' - - return None - - def validate_sectionless_conf(self, file_contents, expected): - """A crude conf parser. Useful to inspect configuration files which - do not have section headers (as would be necessary in order to use - the configparser). Such as openstack-dashboard or rabbitmq confs.""" - for line in file_contents.split('\n'): - if '=' in line: - args = line.split('=') - if len(args) <= 1: - continue - key = args[0].strip() - value = args[1].strip() - if key in expected.keys(): - if expected[key] != value: - msg = ('Config mismatch. Expected, actual: {}, ' - '{}'.format(expected[key], value)) - amulet.raise_status(amulet.FAIL, msg=msg) - - def get_unit_hostnames(self, units): - """Return a dict of juju unit names to hostnames.""" - host_names = {} - for unit in units: - host_names[unit.info['unit_name']] = \ - str(unit.file_contents('/etc/hostname').strip()) - self.log.debug('Unit host names: {}'.format(host_names)) - return host_names - - def run_cmd_unit(self, sentry_unit, cmd): - """Run a command on a unit, return the output and exit code.""" - output, code = sentry_unit.run(cmd) - if code == 0: - self.log.debug('{} `{}` command returned {} ' - '(OK)'.format(sentry_unit.info['unit_name'], - cmd, code)) - else: - msg = ('{} `{}` command returned {} ' - '{}'.format(sentry_unit.info['unit_name'], - cmd, code, output)) - amulet.raise_status(amulet.FAIL, msg=msg) - return str(output), code - - def file_exists_on_unit(self, sentry_unit, file_name): - """Check if a file exists on a unit.""" - try: - sentry_unit.file_stat(file_name) - return True - except IOError: - return False - except Exception as e: - msg = 'Error checking file {}: {}'.format(file_name, e) - amulet.raise_status(amulet.FAIL, msg=msg) - - def file_contents_safe(self, sentry_unit, file_name, - max_wait=60, fatal=False): - """Get file contents from a sentry unit. Wrap amulet file_contents - with retry logic to address races where a file checks as existing, - but no longer exists by the time file_contents is called. - Return None if file not found. Optionally raise if fatal is True.""" - unit_name = sentry_unit.info['unit_name'] - file_contents = False - tries = 0 - while not file_contents and tries < (max_wait / 4): - try: - file_contents = sentry_unit.file_contents(file_name) - except IOError: - self.log.debug('Attempt {} to open file {} from {} ' - 'failed'.format(tries, file_name, - unit_name)) - time.sleep(4) - tries += 1 - - if file_contents: - return file_contents - elif not fatal: - return None - elif fatal: - msg = 'Failed to get file contents from unit.' - amulet.raise_status(amulet.FAIL, msg) - - def port_knock_tcp(self, host="localhost", port=22, timeout=15): - """Open a TCP socket to check for a listening sevice on a host. - - :param host: host name or IP address, default to localhost - :param port: TCP port number, default to 22 - :param timeout: Connect timeout, default to 15 seconds - :returns: True if successful, False if connect failed - """ - - # Resolve host name if possible - try: - connect_host = socket.gethostbyname(host) - host_human = "{} ({})".format(connect_host, host) - except socket.error as e: - self.log.warn('Unable to resolve address: ' - '{} ({}) Trying anyway!'.format(host, e)) - connect_host = host - host_human = connect_host - - # Attempt socket connection - try: - knock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - knock.settimeout(timeout) - knock.connect((connect_host, port)) - knock.close() - self.log.debug('Socket connect OK for host ' - '{} on port {}.'.format(host_human, port)) - return True - except socket.error as e: - self.log.debug('Socket connect FAIL for' - ' {} port {} ({})'.format(host_human, port, e)) - return False - - def port_knock_units(self, sentry_units, port=22, - timeout=15, expect_success=True): - """Open a TCP socket to check for a listening sevice on each - listed juju unit. - - :param sentry_units: list of sentry unit pointers - :param port: TCP port number, default to 22 - :param timeout: Connect timeout, default to 15 seconds - :expect_success: True by default, set False to invert logic - :returns: None if successful, Failure message otherwise - """ - for unit in sentry_units: - host = unit.info['public-address'] - connected = self.port_knock_tcp(host, port, timeout) - if not connected and expect_success: - return 'Socket connect failed.' - elif connected and not expect_success: - return 'Socket connected unexpectedly.' - - def get_uuid_epoch_stamp(self): - """Returns a stamp string based on uuid4 and epoch time. Useful in - generating test messages which need to be unique-ish.""" - return '[{}-{}]'.format(uuid.uuid4(), time.time()) - - # amulet juju action helpers: - def run_action(self, unit_sentry, action, - _check_output=subprocess.check_output, - params=None): - """Translate to amulet's built in run_action(). Deprecated. - - Run the named action on a given unit sentry. - - params a dict of parameters to use - _check_output parameter is no longer used - - @return action_id. - """ - self.log.warn('charmhelpers.contrib.amulet.utils.run_action has been ' - 'deprecated for amulet.run_action') - return unit_sentry.run_action(action, action_args=params) - - def wait_on_action(self, action_id, _check_output=subprocess.check_output): - """Wait for a given action, returning if it completed or not. - - action_id a string action uuid - _check_output parameter is no longer used - """ - data = amulet.actions.get_action_output(action_id, full_output=True) - return data.get(u"status") == "completed" - - def status_get(self, unit): - """Return the current service status of this unit.""" - raw_status, return_code = unit.run( - "status-get --format=json --include-data") - if return_code != 0: - return ("unknown", "") - status = json.loads(raw_status) - return (status["status"], status["message"]) diff --git a/tests/charmhelpers/contrib/openstack/__init__.py b/tests/charmhelpers/contrib/openstack/__init__.py deleted file mode 100644 index d7567b8..0000000 --- a/tests/charmhelpers/contrib/openstack/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tests/charmhelpers/contrib/openstack/amulet/__init__.py b/tests/charmhelpers/contrib/openstack/amulet/__init__.py deleted file mode 100644 index d7567b8..0000000 --- a/tests/charmhelpers/contrib/openstack/amulet/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py deleted file mode 100644 index 5afbbd8..0000000 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ /dev/null @@ -1,354 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import os -import re -import sys -import six -from collections import OrderedDict -from charmhelpers.contrib.amulet.deployment import ( - AmuletDeployment -) - -DEBUG = logging.DEBUG -ERROR = logging.ERROR - - -class OpenStackAmuletDeployment(AmuletDeployment): - """OpenStack amulet deployment. - - This class inherits from AmuletDeployment and has additional support - that is specifically for use by OpenStack charms. - """ - - def __init__(self, series=None, openstack=None, source=None, - stable=True, log_level=DEBUG): - """Initialize the deployment environment.""" - super(OpenStackAmuletDeployment, self).__init__(series) - self.log = self.get_logger(level=log_level) - self.log.info('OpenStackAmuletDeployment: init') - self.openstack = openstack - self.source = source - self.stable = stable - - def get_logger(self, name="deployment-logger", level=logging.DEBUG): - """Get a logger object that will log to stdout.""" - log = logging - logger = log.getLogger(name) - fmt = log.Formatter("%(asctime)s %(funcName)s " - "%(levelname)s: %(message)s") - - handler = log.StreamHandler(stream=sys.stdout) - handler.setLevel(level) - handler.setFormatter(fmt) - - logger.addHandler(handler) - logger.setLevel(level) - - return logger - - def _determine_branch_locations(self, other_services): - """Determine the branch locations for the other services. - - Determine if the local branch being tested is derived from its - stable or next (dev) branch, and based on this, use the corresonding - stable or next branches for the other_services.""" - - self.log.info('OpenStackAmuletDeployment: determine branch locations') - - # Charms outside the ~openstack-charmers - base_charms = { - 'mysql': ['trusty'], - 'mongodb': ['trusty'], - 'nrpe': ['trusty', 'xenial'], - } - - for svc in other_services: - # If a location has been explicitly set, use it - if svc.get('location'): - continue - if svc['name'] in base_charms: - # NOTE: not all charms have support for all series we - # want/need to test against, so fix to most recent - # that each base charm supports - target_series = self.series - if self.series not in base_charms[svc['name']]: - target_series = base_charms[svc['name']][-1] - svc['location'] = 'cs:{}/{}'.format(target_series, - svc['name']) - elif self.stable: - svc['location'] = 'cs:{}/{}'.format(self.series, - svc['name']) - else: - svc['location'] = 'cs:~openstack-charmers-next/{}/{}'.format( - self.series, - svc['name'] - ) - - return other_services - - def _add_services(self, this_service, other_services, use_source=None, - no_origin=None): - """Add services to the deployment and optionally set - openstack-origin/source. - - :param this_service dict: Service dictionary describing the service - whose amulet tests are being run - :param other_services dict: List of service dictionaries describing - the services needed to support the target - service - :param use_source list: List of services which use the 'source' config - option rather than 'openstack-origin' - :param no_origin list: List of services which do not support setting - the Cloud Archive. - Service Dict: - { - 'name': str charm-name, - 'units': int number of units, - 'constraints': dict of juju constraints, - 'location': str location of charm, - } - eg - this_service = { - 'name': 'openvswitch-odl', - 'constraints': {'mem': '8G'}, - } - other_services = [ - { - 'name': 'nova-compute', - 'units': 2, - 'constraints': {'mem': '4G'}, - 'location': cs:~bob/xenial/nova-compute - }, - { - 'name': 'mysql', - 'constraints': {'mem': '2G'}, - }, - {'neutron-api-odl'}] - use_source = ['mysql'] - no_origin = ['neutron-api-odl'] - """ - self.log.info('OpenStackAmuletDeployment: adding services') - - other_services = self._determine_branch_locations(other_services) - - super(OpenStackAmuletDeployment, self)._add_services(this_service, - other_services) - - services = other_services - services.append(this_service) - - use_source = use_source or [] - no_origin = no_origin or [] - - # Charms which should use the source config option - use_source = list(set( - use_source + ['mysql', 'mongodb', 'rabbitmq-server', 'ceph', - 'ceph-osd', 'ceph-radosgw', 'ceph-mon', - 'ceph-proxy', 'percona-cluster', 'lxd'])) - - # Charms which can not use openstack-origin, ie. many subordinates - no_origin = list(set( - no_origin + ['cinder-ceph', 'hacluster', 'neutron-openvswitch', - 'nrpe', 'openvswitch-odl', 'neutron-api-odl', - 'odl-controller', 'cinder-backup', 'nexentaedge-data', - 'nexentaedge-iscsi-gw', 'nexentaedge-swift-gw', - 'cinder-nexentaedge', 'nexentaedge-mgmt'])) - - if self.openstack: - for svc in services: - if svc['name'] not in use_source + no_origin: - config = {'openstack-origin': self.openstack} - self.d.configure(svc['name'], config) - - if self.source: - for svc in services: - if svc['name'] in use_source and svc['name'] not in no_origin: - config = {'source': self.source} - self.d.configure(svc['name'], config) - - def _configure_services(self, configs): - """Configure all of the services.""" - self.log.info('OpenStackAmuletDeployment: configure services') - for service, config in six.iteritems(configs): - self.d.configure(service, config) - - def _auto_wait_for_status(self, message=None, exclude_services=None, - include_only=None, timeout=None): - """Wait for all units to have a specific extended status, except - for any defined as excluded. Unless specified via message, any - status containing any case of 'ready' will be considered a match. - - Examples of message usage: - - Wait for all unit status to CONTAIN any case of 'ready' or 'ok': - message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE) - - Wait for all units to reach this status (exact match): - message = re.compile('^Unit is ready and clustered$') - - Wait for all units to reach any one of these (exact match): - message = re.compile('Unit is ready|OK|Ready') - - Wait for at least one unit to reach this status (exact match): - message = {'ready'} - - See Amulet's sentry.wait_for_messages() for message usage detail. - https://github.com/juju/amulet/blob/master/amulet/sentry.py - - :param message: Expected status match - :param exclude_services: List of juju service names to ignore, - not to be used in conjuction with include_only. - :param include_only: List of juju service names to exclusively check, - not to be used in conjuction with exclude_services. - :param timeout: Maximum time in seconds to wait for status match - :returns: None. Raises if timeout is hit. - """ - if not timeout: - timeout = int(os.environ.get('AMULET_SETUP_TIMEOUT', 1800)) - self.log.info('Waiting for extended status on units for {}s...' - ''.format(timeout)) - - all_services = self.d.services.keys() - - if exclude_services and include_only: - raise ValueError('exclude_services can not be used ' - 'with include_only') - - if message: - if isinstance(message, re._pattern_type): - match = message.pattern - else: - match = message - - self.log.debug('Custom extended status wait match: ' - '{}'.format(match)) - else: - self.log.debug('Default extended status wait match: contains ' - 'READY (case-insensitive)') - message = re.compile('.*ready.*', re.IGNORECASE) - - if exclude_services: - self.log.debug('Excluding services from extended status match: ' - '{}'.format(exclude_services)) - else: - exclude_services = [] - - if include_only: - services = include_only - else: - services = list(set(all_services) - set(exclude_services)) - - self.log.debug('Waiting up to {}s for extended status on services: ' - '{}'.format(timeout, services)) - service_messages = {service: message for service in services} - - # Check for idleness - self.d.sentry.wait(timeout=timeout) - # Check for error states and bail early - self.d.sentry.wait_for_status(self.d.juju_env, services, timeout=timeout) - # Check for ready messages - self.d.sentry.wait_for_messages(service_messages, timeout=timeout) - - self.log.info('OK') - - def _get_openstack_release(self): - """Get openstack release. - - Return an integer representing the enum value of the openstack - release. - """ - # Must be ordered by OpenStack release (not by Ubuntu release): - (self.trusty_icehouse, self.trusty_kilo, self.trusty_liberty, - self.trusty_mitaka, self.xenial_mitaka, self.xenial_newton, - self.yakkety_newton, self.xenial_ocata, self.zesty_ocata, - self.xenial_pike, self.artful_pike, self.xenial_queens, - self.bionic_queens,) = range(13) - - releases = { - ('trusty', None): self.trusty_icehouse, - ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo, - ('trusty', 'cloud:trusty-liberty'): self.trusty_liberty, - ('trusty', 'cloud:trusty-mitaka'): self.trusty_mitaka, - ('xenial', None): self.xenial_mitaka, - ('xenial', 'cloud:xenial-newton'): self.xenial_newton, - ('xenial', 'cloud:xenial-ocata'): self.xenial_ocata, - ('xenial', 'cloud:xenial-pike'): self.xenial_pike, - ('xenial', 'cloud:xenial-queens'): self.xenial_queens, - ('yakkety', None): self.yakkety_newton, - ('zesty', None): self.zesty_ocata, - ('artful', None): self.artful_pike, - ('bionic', None): self.bionic_queens, - } - return releases[(self.series, self.openstack)] - - def _get_openstack_release_string(self): - """Get openstack release string. - - Return a string representing the openstack release. - """ - releases = OrderedDict([ - ('trusty', 'icehouse'), - ('xenial', 'mitaka'), - ('yakkety', 'newton'), - ('zesty', 'ocata'), - ('artful', 'pike'), - ('bionic', 'queens'), - ]) - if self.openstack: - os_origin = self.openstack.split(':')[1] - return os_origin.split('%s-' % self.series)[1].split('/')[0] - else: - return releases[self.series] - - def get_ceph_expected_pools(self, radosgw=False): - """Return a list of expected ceph pools in a ceph + cinder + glance - test scenario, based on OpenStack release and whether ceph radosgw - is flagged as present or not.""" - - if self._get_openstack_release() == self.trusty_icehouse: - # Icehouse - pools = [ - 'data', - 'metadata', - 'rbd', - 'cinder-ceph', - 'glance' - ] - elif (self.trusty_kilo <= self._get_openstack_release() <= - self.zesty_ocata): - # Kilo through Ocata - pools = [ - 'rbd', - 'cinder-ceph', - 'glance' - ] - else: - # Pike and later - pools = [ - 'cinder-ceph', - 'glance' - ] - - if radosgw: - pools.extend([ - '.rgw.root', - '.rgw.control', - '.rgw', - '.rgw.gc', - '.users.uid' - ]) - - return pools diff --git a/tests/charmhelpers/contrib/openstack/amulet/utils.py b/tests/charmhelpers/contrib/openstack/amulet/utils.py deleted file mode 100644 index 87f364d..0000000 --- a/tests/charmhelpers/contrib/openstack/amulet/utils.py +++ /dev/null @@ -1,1338 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import amulet -import json -import logging -import os -import re -import six -import time -import urllib -import urlparse - -import cinderclient.v1.client as cinder_client -import cinderclient.v2.client as cinder_clientv2 -import glanceclient.v1.client as glance_client -import heatclient.v1.client as heat_client -from keystoneclient.v2_0 import client as keystone_client -from keystoneauth1.identity import ( - v3, - v2, -) -from keystoneauth1 import session as keystone_session -from keystoneclient.v3 import client as keystone_client_v3 -from novaclient import exceptions - -import novaclient.client as nova_client -import novaclient -import pika -import swiftclient - -from charmhelpers.contrib.amulet.utils import ( - AmuletUtils -) -from charmhelpers.core.host import CompareHostReleases - -DEBUG = logging.DEBUG -ERROR = logging.ERROR - -NOVA_CLIENT_VERSION = "2" - - -class OpenStackAmuletUtils(AmuletUtils): - """OpenStack amulet utilities. - - This class inherits from AmuletUtils and has additional support - that is specifically for use by OpenStack charm tests. - """ - - def __init__(self, log_level=ERROR): - """Initialize the deployment environment.""" - super(OpenStackAmuletUtils, self).__init__(log_level) - - def validate_endpoint_data(self, endpoints, admin_port, internal_port, - public_port, expected): - """Validate endpoint data. - - Validate actual endpoint data vs expected endpoint data. The ports - are used to find the matching endpoint. - """ - self.log.debug('Validating endpoint data...') - self.log.debug('actual: {}'.format(repr(endpoints))) - found = False - for ep in endpoints: - self.log.debug('endpoint: {}'.format(repr(ep))) - if (admin_port in ep.adminurl and - internal_port in ep.internalurl and - public_port in ep.publicurl): - found = True - actual = {'id': ep.id, - 'region': ep.region, - 'adminurl': ep.adminurl, - 'internalurl': ep.internalurl, - 'publicurl': ep.publicurl, - 'service_id': ep.service_id} - ret = self._validate_dict_data(expected, actual) - if ret: - return 'unexpected endpoint data - {}'.format(ret) - - if not found: - return 'endpoint not found' - - def validate_v3_endpoint_data(self, endpoints, admin_port, internal_port, - public_port, expected): - """Validate keystone v3 endpoint data. - - Validate the v3 endpoint data which has changed from v2. The - ports are used to find the matching endpoint. - - The new v3 endpoint data looks like: - - ['}, - region=RegionOne, - region_id=RegionOne, - service_id=17f842a0dc084b928e476fafe67e4095, - url=http://10.5.6.5:9312>, - '}, - region=RegionOne, - region_id=RegionOne, - service_id=72fc8736fb41435e8b3584205bb2cfa3, - url=http://10.5.6.6:35357/v3>, - ... ] - """ - self.log.debug('Validating v3 endpoint data...') - self.log.debug('actual: {}'.format(repr(endpoints))) - found = [] - for ep in endpoints: - self.log.debug('endpoint: {}'.format(repr(ep))) - if ((admin_port in ep.url and ep.interface == 'admin') or - (internal_port in ep.url and ep.interface == 'internal') or - (public_port in ep.url and ep.interface == 'public')): - found.append(ep.interface) - # note we ignore the links member. - actual = {'id': ep.id, - 'region': ep.region, - 'region_id': ep.region_id, - 'interface': self.not_null, - 'url': ep.url, - 'service_id': ep.service_id, } - ret = self._validate_dict_data(expected, actual) - if ret: - return 'unexpected endpoint data - {}'.format(ret) - - if len(found) != 3: - return 'Unexpected number of endpoints found' - - def validate_svc_catalog_endpoint_data(self, expected, actual): - """Validate service catalog endpoint data. - - Validate a list of actual service catalog endpoints vs a list of - expected service catalog endpoints. - """ - self.log.debug('Validating service catalog endpoint data...') - self.log.debug('actual: {}'.format(repr(actual))) - for k, v in six.iteritems(expected): - if k in actual: - ret = self._validate_dict_data(expected[k][0], actual[k][0]) - if ret: - return self.endpoint_error(k, ret) - else: - return "endpoint {} does not exist".format(k) - return ret - - def validate_v3_svc_catalog_endpoint_data(self, expected, actual): - """Validate the keystone v3 catalog endpoint data. - - Validate a list of dictinaries that make up the keystone v3 service - catalogue. - - It is in the form of: - - - {u'identity': [{u'id': u'48346b01c6804b298cdd7349aadb732e', - u'interface': u'admin', - u'region': u'RegionOne', - u'region_id': u'RegionOne', - u'url': u'http://10.5.5.224:35357/v3'}, - {u'id': u'8414f7352a4b47a69fddd9dbd2aef5cf', - u'interface': u'public', - u'region': u'RegionOne', - u'region_id': u'RegionOne', - u'url': u'http://10.5.5.224:5000/v3'}, - {u'id': u'd5ca31440cc24ee1bf625e2996fb6a5b', - u'interface': u'internal', - u'region': u'RegionOne', - u'region_id': u'RegionOne', - u'url': u'http://10.5.5.224:5000/v3'}], - u'key-manager': [{u'id': u'68ebc17df0b045fcb8a8a433ebea9e62', - u'interface': u'public', - u'region': u'RegionOne', - u'region_id': u'RegionOne', - u'url': u'http://10.5.5.223:9311'}, - {u'id': u'9cdfe2a893c34afd8f504eb218cd2f9d', - u'interface': u'internal', - u'region': u'RegionOne', - u'region_id': u'RegionOne', - u'url': u'http://10.5.5.223:9311'}, - {u'id': u'f629388955bc407f8b11d8b7ca168086', - u'interface': u'admin', - u'region': u'RegionOne', - u'region_id': u'RegionOne', - u'url': u'http://10.5.5.223:9312'}]} - - Note, that an added complication is that the order of admin, public, - internal against 'interface' in each region. - - Thus, the function sorts the expected and actual lists using the - interface key as a sort key, prior to the comparison. - """ - self.log.debug('Validating v3 service catalog endpoint data...') - self.log.debug('actual: {}'.format(repr(actual))) - for k, v in six.iteritems(expected): - if k in actual: - l_expected = sorted(v, key=lambda x: x['interface']) - l_actual = sorted(actual[k], key=lambda x: x['interface']) - if len(l_actual) != len(l_expected): - return ("endpoint {} has differing number of interfaces " - " - expected({}), actual({})" - .format(k, len(l_expected), len(l_actual))) - for i_expected, i_actual in zip(l_expected, l_actual): - self.log.debug("checking interface {}" - .format(i_expected['interface'])) - ret = self._validate_dict_data(i_expected, i_actual) - if ret: - return self.endpoint_error(k, ret) - else: - return "endpoint {} does not exist".format(k) - return ret - - def validate_tenant_data(self, expected, actual): - """Validate tenant data. - - Validate a list of actual tenant data vs list of expected tenant - data. - """ - self.log.debug('Validating tenant data...') - self.log.debug('actual: {}'.format(repr(actual))) - for e in expected: - found = False - for act in actual: - a = {'enabled': act.enabled, 'description': act.description, - 'name': act.name, 'id': act.id} - if e['name'] == a['name']: - found = True - ret = self._validate_dict_data(e, a) - if ret: - return "unexpected tenant data - {}".format(ret) - if not found: - return "tenant {} does not exist".format(e['name']) - return ret - - def validate_role_data(self, expected, actual): - """Validate role data. - - Validate a list of actual role data vs a list of expected role - data. - """ - self.log.debug('Validating role data...') - self.log.debug('actual: {}'.format(repr(actual))) - for e in expected: - found = False - for act in actual: - a = {'name': act.name, 'id': act.id} - if e['name'] == a['name']: - found = True - ret = self._validate_dict_data(e, a) - if ret: - return "unexpected role data - {}".format(ret) - if not found: - return "role {} does not exist".format(e['name']) - return ret - - def validate_user_data(self, expected, actual, api_version=None): - """Validate user data. - - Validate a list of actual user data vs a list of expected user - data. - """ - self.log.debug('Validating user data...') - self.log.debug('actual: {}'.format(repr(actual))) - for e in expected: - found = False - for act in actual: - if e['name'] == act.name: - a = {'enabled': act.enabled, 'name': act.name, - 'email': act.email, 'id': act.id} - if api_version == 3: - a['default_project_id'] = getattr(act, - 'default_project_id', - 'none') - else: - a['tenantId'] = act.tenantId - found = True - ret = self._validate_dict_data(e, a) - if ret: - return "unexpected user data - {}".format(ret) - if not found: - return "user {} does not exist".format(e['name']) - return ret - - def validate_flavor_data(self, expected, actual): - """Validate flavor data. - - Validate a list of actual flavors vs a list of expected flavors. - """ - self.log.debug('Validating flavor data...') - self.log.debug('actual: {}'.format(repr(actual))) - act = [a.name for a in actual] - return self._validate_list_data(expected, act) - - def tenant_exists(self, keystone, tenant): - """Return True if tenant exists.""" - self.log.debug('Checking if tenant exists ({})...'.format(tenant)) - return tenant in [t.name for t in keystone.tenants.list()] - - def keystone_wait_for_propagation(self, sentry_relation_pairs, - api_version): - """Iterate over list of sentry and relation tuples and verify that - api_version has the expected value. - - :param sentry_relation_pairs: list of sentry, relation name tuples used - for monitoring propagation of relation - data - :param api_version: api_version to expect in relation data - :returns: None if successful. Raise on error. - """ - for (sentry, relation_name) in sentry_relation_pairs: - rel = sentry.relation('identity-service', - relation_name) - self.log.debug('keystone relation data: {}'.format(rel)) - if rel.get('api_version') != str(api_version): - raise Exception("api_version not propagated through relation" - " data yet ('{}' != '{}')." - "".format(rel['api_version'], api_version)) - - def keystone_configure_api_version(self, sentry_relation_pairs, deployment, - api_version): - """Configure preferred-api-version of keystone in deployment and - monitor provided list of relation objects for propagation - before returning to caller. - - :param sentry_relation_pairs: list of sentry, relation tuples used for - monitoring propagation of relation data - :param deployment: deployment to configure - :param api_version: value preferred-api-version will be set to - :returns: None if successful. Raise on error. - """ - self.log.debug("Setting keystone preferred-api-version: '{}'" - "".format(api_version)) - - config = {'preferred-api-version': api_version} - deployment.d.configure('keystone', config) - deployment._auto_wait_for_status() - self.keystone_wait_for_propagation(sentry_relation_pairs, api_version) - - def authenticate_cinder_admin(self, keystone_sentry, username, - password, tenant, api_version=2): - """Authenticates admin user with cinder.""" - # NOTE(beisner): cinder python client doesn't accept tokens. - keystone_ip = keystone_sentry.info['public-address'] - ept = "http://{}:5000/v2.0".format(keystone_ip.strip().decode('utf-8')) - _clients = { - 1: cinder_client.Client, - 2: cinder_clientv2.Client} - return _clients[api_version](username, password, tenant, ept) - - def authenticate_keystone(self, keystone_ip, username, password, - api_version=False, admin_port=False, - user_domain_name=None, domain_name=None, - project_domain_name=None, project_name=None): - """Authenticate with Keystone""" - self.log.debug('Authenticating with keystone...') - port = 5000 - if admin_port: - port = 35357 - base_ep = "http://{}:{}".format(keystone_ip.strip().decode('utf-8'), - port) - if not api_version or api_version == 2: - ep = base_ep + "/v2.0" - auth = v2.Password( - username=username, - password=password, - tenant_name=project_name, - auth_url=ep - ) - sess = keystone_session.Session(auth=auth) - client = keystone_client.Client(session=sess) - # This populates the client.service_catalog - client.auth_ref = auth.get_access(sess) - return client - else: - ep = base_ep + "/v3" - auth = v3.Password( - user_domain_name=user_domain_name, - username=username, - password=password, - domain_name=domain_name, - project_domain_name=project_domain_name, - project_name=project_name, - auth_url=ep - ) - sess = keystone_session.Session(auth=auth) - client = keystone_client_v3.Client(session=sess) - # This populates the client.service_catalog - client.auth_ref = auth.get_access(sess) - return client - - def authenticate_keystone_admin(self, keystone_sentry, user, password, - tenant=None, api_version=None, - keystone_ip=None, user_domain_name=None, - project_domain_name=None, - project_name=None): - """Authenticates admin user with the keystone admin endpoint.""" - self.log.debug('Authenticating keystone admin...') - if not keystone_ip: - keystone_ip = keystone_sentry.info['public-address'] - - # To support backward compatibility usage of this function - if not project_name: - project_name = tenant - if api_version == 3 and not user_domain_name: - user_domain_name = 'admin_domain' - if api_version == 3 and not project_domain_name: - project_domain_name = 'admin_domain' - if api_version == 3 and not project_name: - project_name = 'admin' - - return self.authenticate_keystone( - keystone_ip, user, password, - api_version=api_version, - user_domain_name=user_domain_name, - project_domain_name=project_domain_name, - project_name=project_name, - admin_port=True) - - def authenticate_keystone_user(self, keystone, user, password, tenant): - """Authenticates a regular user with the keystone public endpoint.""" - self.log.debug('Authenticating keystone user ({})...'.format(user)) - ep = keystone.service_catalog.url_for(service_type='identity', - interface='publicURL') - keystone_ip = urlparse.urlparse(ep).hostname - - return self.authenticate_keystone(keystone_ip, user, password, - project_name=tenant) - - def authenticate_glance_admin(self, keystone): - """Authenticates admin user with glance.""" - self.log.debug('Authenticating glance admin...') - ep = keystone.service_catalog.url_for(service_type='image', - interface='adminURL') - if keystone.session: - return glance_client.Client(ep, session=keystone.session) - else: - return glance_client.Client(ep, token=keystone.auth_token) - - def authenticate_heat_admin(self, keystone): - """Authenticates the admin user with heat.""" - self.log.debug('Authenticating heat admin...') - ep = keystone.service_catalog.url_for(service_type='orchestration', - interface='publicURL') - if keystone.session: - return heat_client.Client(endpoint=ep, session=keystone.session) - else: - return heat_client.Client(endpoint=ep, token=keystone.auth_token) - - def authenticate_nova_user(self, keystone, user, password, tenant): - """Authenticates a regular user with nova-api.""" - self.log.debug('Authenticating nova user ({})...'.format(user)) - ep = keystone.service_catalog.url_for(service_type='identity', - interface='publicURL') - if keystone.session: - return nova_client.Client(NOVA_CLIENT_VERSION, - session=keystone.session, - auth_url=ep) - elif novaclient.__version__[0] >= "7": - return nova_client.Client(NOVA_CLIENT_VERSION, - username=user, password=password, - project_name=tenant, auth_url=ep) - else: - return nova_client.Client(NOVA_CLIENT_VERSION, - username=user, api_key=password, - project_id=tenant, auth_url=ep) - - def authenticate_swift_user(self, keystone, user, password, tenant): - """Authenticates a regular user with swift api.""" - self.log.debug('Authenticating swift user ({})...'.format(user)) - ep = keystone.service_catalog.url_for(service_type='identity', - interface='publicURL') - if keystone.session: - return swiftclient.Connection(session=keystone.session) - else: - return swiftclient.Connection(authurl=ep, - user=user, - key=password, - tenant_name=tenant, - auth_version='2.0') - - def create_flavor(self, nova, name, ram, vcpus, disk, flavorid="auto", - ephemeral=0, swap=0, rxtx_factor=1.0, is_public=True): - """Create the specified flavor.""" - try: - nova.flavors.find(name=name) - except (exceptions.NotFound, exceptions.NoUniqueMatch): - self.log.debug('Creating flavor ({})'.format(name)) - nova.flavors.create(name, ram, vcpus, disk, flavorid, - ephemeral, swap, rxtx_factor, is_public) - - def create_cirros_image(self, glance, image_name): - """Download the latest cirros image and upload it to glance, - validate and return a resource pointer. - - :param glance: pointer to authenticated glance connection - :param image_name: display name for new image - :returns: glance image pointer - """ - self.log.debug('Creating glance cirros image ' - '({})...'.format(image_name)) - - # Download cirros image - http_proxy = os.getenv('AMULET_HTTP_PROXY') - self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy)) - if http_proxy: - proxies = {'http': http_proxy} - opener = urllib.FancyURLopener(proxies) - else: - opener = urllib.FancyURLopener() - - f = opener.open('http://download.cirros-cloud.net/version/released') - version = f.read().strip() - cirros_img = 'cirros-{}-x86_64-disk.img'.format(version) - local_path = os.path.join('tests', cirros_img) - - if not os.path.exists(local_path): - cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net', - version, cirros_img) - opener.retrieve(cirros_url, local_path) - f.close() - - # Create glance image - with open(local_path) as f: - image = glance.images.create(name=image_name, is_public=True, - disk_format='qcow2', - container_format='bare', data=f) - - # Wait for image to reach active status - img_id = image.id - ret = self.resource_reaches_status(glance.images, img_id, - expected_stat='active', - msg='Image status wait') - if not ret: - msg = 'Glance image failed to reach expected state.' - amulet.raise_status(amulet.FAIL, msg=msg) - - # Re-validate new image - self.log.debug('Validating image attributes...') - val_img_name = glance.images.get(img_id).name - val_img_stat = glance.images.get(img_id).status - val_img_pub = glance.images.get(img_id).is_public - val_img_cfmt = glance.images.get(img_id).container_format - val_img_dfmt = glance.images.get(img_id).disk_format - msg_attr = ('Image attributes - name:{} public:{} id:{} stat:{} ' - 'container fmt:{} disk fmt:{}'.format( - val_img_name, val_img_pub, img_id, - val_img_stat, val_img_cfmt, val_img_dfmt)) - - if val_img_name == image_name and val_img_stat == 'active' \ - and val_img_pub is True and val_img_cfmt == 'bare' \ - and val_img_dfmt == 'qcow2': - self.log.debug(msg_attr) - else: - msg = ('Volume validation failed, {}'.format(msg_attr)) - amulet.raise_status(amulet.FAIL, msg=msg) - - return image - - def delete_image(self, glance, image): - """Delete the specified image.""" - - # /!\ DEPRECATION WARNING - self.log.warn('/!\\ DEPRECATION WARNING: use ' - 'delete_resource instead of delete_image.') - self.log.debug('Deleting glance image ({})...'.format(image)) - return self.delete_resource(glance.images, image, msg='glance image') - - def create_instance(self, nova, image_name, instance_name, flavor): - """Create the specified instance.""" - self.log.debug('Creating instance ' - '({}|{}|{})'.format(instance_name, image_name, flavor)) - image = nova.glance.find_image(image_name) - flavor = nova.flavors.find(name=flavor) - instance = nova.servers.create(name=instance_name, image=image, - flavor=flavor) - - count = 1 - status = instance.status - while status != 'ACTIVE' and count < 60: - time.sleep(3) - instance = nova.servers.get(instance.id) - status = instance.status - self.log.debug('instance status: {}'.format(status)) - count += 1 - - if status != 'ACTIVE': - self.log.error('instance creation timed out') - return None - - return instance - - def delete_instance(self, nova, instance): - """Delete the specified instance.""" - - # /!\ DEPRECATION WARNING - self.log.warn('/!\\ DEPRECATION WARNING: use ' - 'delete_resource instead of delete_instance.') - self.log.debug('Deleting instance ({})...'.format(instance)) - return self.delete_resource(nova.servers, instance, - msg='nova instance') - - def create_or_get_keypair(self, nova, keypair_name="testkey"): - """Create a new keypair, or return pointer if it already exists.""" - try: - _keypair = nova.keypairs.get(keypair_name) - self.log.debug('Keypair ({}) already exists, ' - 'using it.'.format(keypair_name)) - return _keypair - except Exception: - self.log.debug('Keypair ({}) does not exist, ' - 'creating it.'.format(keypair_name)) - - _keypair = nova.keypairs.create(name=keypair_name) - return _keypair - - def _get_cinder_obj_name(self, cinder_object): - """Retrieve name of cinder object. - - :param cinder_object: cinder snapshot or volume object - :returns: str cinder object name - """ - # v1 objects store name in 'display_name' attr but v2+ use 'name' - try: - return cinder_object.display_name - except AttributeError: - return cinder_object.name - - def create_cinder_volume(self, cinder, vol_name="demo-vol", vol_size=1, - img_id=None, src_vol_id=None, snap_id=None): - """Create cinder volume, optionally from a glance image, OR - optionally as a clone of an existing volume, OR optionally - from a snapshot. Wait for the new volume status to reach - the expected status, validate and return a resource pointer. - - :param vol_name: cinder volume display name - :param vol_size: size in gigabytes - :param img_id: optional glance image id - :param src_vol_id: optional source volume id to clone - :param snap_id: optional snapshot id to use - :returns: cinder volume pointer - """ - # Handle parameter input and avoid impossible combinations - if img_id and not src_vol_id and not snap_id: - # Create volume from image - self.log.debug('Creating cinder volume from glance image...') - bootable = 'true' - elif src_vol_id and not img_id and not snap_id: - # Clone an existing volume - self.log.debug('Cloning cinder volume...') - bootable = cinder.volumes.get(src_vol_id).bootable - elif snap_id and not src_vol_id and not img_id: - # Create volume from snapshot - self.log.debug('Creating cinder volume from snapshot...') - snap = cinder.volume_snapshots.find(id=snap_id) - vol_size = snap.size - snap_vol_id = cinder.volume_snapshots.get(snap_id).volume_id - bootable = cinder.volumes.get(snap_vol_id).bootable - elif not img_id and not src_vol_id and not snap_id: - # Create volume - self.log.debug('Creating cinder volume...') - bootable = 'false' - else: - # Impossible combination of parameters - msg = ('Invalid method use - name:{} size:{} img_id:{} ' - 'src_vol_id:{} snap_id:{}'.format(vol_name, vol_size, - img_id, src_vol_id, - snap_id)) - amulet.raise_status(amulet.FAIL, msg=msg) - - # Create new volume - try: - vol_new = cinder.volumes.create(display_name=vol_name, - imageRef=img_id, - size=vol_size, - source_volid=src_vol_id, - snapshot_id=snap_id) - vol_id = vol_new.id - except TypeError: - vol_new = cinder.volumes.create(name=vol_name, - imageRef=img_id, - size=vol_size, - source_volid=src_vol_id, - snapshot_id=snap_id) - vol_id = vol_new.id - except Exception as e: - msg = 'Failed to create volume: {}'.format(e) - amulet.raise_status(amulet.FAIL, msg=msg) - - # Wait for volume to reach available status - ret = self.resource_reaches_status(cinder.volumes, vol_id, - expected_stat="available", - msg="Volume status wait") - if not ret: - msg = 'Cinder volume failed to reach expected state.' - amulet.raise_status(amulet.FAIL, msg=msg) - - # Re-validate new volume - self.log.debug('Validating volume attributes...') - val_vol_name = self._get_cinder_obj_name(cinder.volumes.get(vol_id)) - val_vol_boot = cinder.volumes.get(vol_id).bootable - val_vol_stat = cinder.volumes.get(vol_id).status - val_vol_size = cinder.volumes.get(vol_id).size - msg_attr = ('Volume attributes - name:{} id:{} stat:{} boot:' - '{} size:{}'.format(val_vol_name, vol_id, - val_vol_stat, val_vol_boot, - val_vol_size)) - - if val_vol_boot == bootable and val_vol_stat == 'available' \ - and val_vol_name == vol_name and val_vol_size == vol_size: - self.log.debug(msg_attr) - else: - msg = ('Volume validation failed, {}'.format(msg_attr)) - amulet.raise_status(amulet.FAIL, msg=msg) - - return vol_new - - def delete_resource(self, resource, resource_id, - msg="resource", max_wait=120): - """Delete one openstack resource, such as one instance, keypair, - image, volume, stack, etc., and confirm deletion within max wait time. - - :param resource: pointer to os resource type, ex:glance_client.images - :param resource_id: unique name or id for the openstack resource - :param msg: text to identify purpose in logging - :param max_wait: maximum wait time in seconds - :returns: True if successful, otherwise False - """ - self.log.debug('Deleting OpenStack resource ' - '{} ({})'.format(resource_id, msg)) - num_before = len(list(resource.list())) - resource.delete(resource_id) - - tries = 0 - num_after = len(list(resource.list())) - while num_after != (num_before - 1) and tries < (max_wait / 4): - self.log.debug('{} delete check: ' - '{} [{}:{}] {}'.format(msg, tries, - num_before, - num_after, - resource_id)) - time.sleep(4) - num_after = len(list(resource.list())) - tries += 1 - - self.log.debug('{}: expected, actual count = {}, ' - '{}'.format(msg, num_before - 1, num_after)) - - if num_after == (num_before - 1): - return True - else: - self.log.error('{} delete timed out'.format(msg)) - return False - - def resource_reaches_status(self, resource, resource_id, - expected_stat='available', - msg='resource', max_wait=120): - """Wait for an openstack resources status to reach an - expected status within a specified time. Useful to confirm that - nova instances, cinder vols, snapshots, glance images, heat stacks - and other resources eventually reach the expected status. - - :param resource: pointer to os resource type, ex: heat_client.stacks - :param resource_id: unique id for the openstack resource - :param expected_stat: status to expect resource to reach - :param msg: text to identify purpose in logging - :param max_wait: maximum wait time in seconds - :returns: True if successful, False if status is not reached - """ - - tries = 0 - resource_stat = resource.get(resource_id).status - while resource_stat != expected_stat and tries < (max_wait / 4): - self.log.debug('{} status check: ' - '{} [{}:{}] {}'.format(msg, tries, - resource_stat, - expected_stat, - resource_id)) - time.sleep(4) - resource_stat = resource.get(resource_id).status - tries += 1 - - self.log.debug('{}: expected, actual status = {}, ' - '{}'.format(msg, resource_stat, expected_stat)) - - if resource_stat == expected_stat: - return True - else: - self.log.debug('{} never reached expected status: ' - '{}'.format(resource_id, expected_stat)) - return False - - def get_ceph_osd_id_cmd(self, index): - """Produce a shell command that will return a ceph-osd id.""" - return ("`initctl list | grep 'ceph-osd ' | " - "awk 'NR=={} {{ print $2 }}' | " - "grep -o '[0-9]*'`".format(index + 1)) - - def get_ceph_pools(self, sentry_unit): - """Return a dict of ceph pools from a single ceph unit, with - pool name as keys, pool id as vals.""" - pools = {} - cmd = 'sudo ceph osd lspools' - output, code = sentry_unit.run(cmd) - if code != 0: - msg = ('{} `{}` returned {} ' - '{}'.format(sentry_unit.info['unit_name'], - cmd, code, output)) - amulet.raise_status(amulet.FAIL, msg=msg) - - # Example output: 0 data,1 metadata,2 rbd,3 cinder,4 glance, - for pool in str(output).split(','): - pool_id_name = pool.split(' ') - if len(pool_id_name) == 2: - pool_id = pool_id_name[0] - pool_name = pool_id_name[1] - pools[pool_name] = int(pool_id) - - self.log.debug('Pools on {}: {}'.format(sentry_unit.info['unit_name'], - pools)) - return pools - - def get_ceph_df(self, sentry_unit): - """Return dict of ceph df json output, including ceph pool state. - - :param sentry_unit: Pointer to amulet sentry instance (juju unit) - :returns: Dict of ceph df output - """ - cmd = 'sudo ceph df --format=json' - output, code = sentry_unit.run(cmd) - if code != 0: - msg = ('{} `{}` returned {} ' - '{}'.format(sentry_unit.info['unit_name'], - cmd, code, output)) - amulet.raise_status(amulet.FAIL, msg=msg) - return json.loads(output) - - def get_ceph_pool_sample(self, sentry_unit, pool_id=0): - """Take a sample of attributes of a ceph pool, returning ceph - pool name, object count and disk space used for the specified - pool ID number. - - :param sentry_unit: Pointer to amulet sentry instance (juju unit) - :param pool_id: Ceph pool ID - :returns: List of pool name, object count, kb disk space used - """ - df = self.get_ceph_df(sentry_unit) - for pool in df['pools']: - if pool['id'] == pool_id: - pool_name = pool['name'] - obj_count = pool['stats']['objects'] - kb_used = pool['stats']['kb_used'] - - self.log.debug('Ceph {} pool (ID {}): {} objects, ' - '{} kb used'.format(pool_name, pool_id, - obj_count, kb_used)) - return pool_name, obj_count, kb_used - - def validate_ceph_pool_samples(self, samples, sample_type="resource pool"): - """Validate ceph pool samples taken over time, such as pool - object counts or pool kb used, before adding, after adding, and - after deleting items which affect those pool attributes. The - 2nd element is expected to be greater than the 1st; 3rd is expected - to be less than the 2nd. - - :param samples: List containing 3 data samples - :param sample_type: String for logging and usage context - :returns: None if successful, Failure message otherwise - """ - original, created, deleted = range(3) - if samples[created] <= samples[original] or \ - samples[deleted] >= samples[created]: - return ('Ceph {} samples ({}) ' - 'unexpected.'.format(sample_type, samples)) - else: - self.log.debug('Ceph {} samples (OK): ' - '{}'.format(sample_type, samples)) - return None - - # rabbitmq/amqp specific helpers: - - def rmq_wait_for_cluster(self, deployment, init_sleep=15, timeout=1200): - """Wait for rmq units extended status to show cluster readiness, - after an optional initial sleep period. Initial sleep is likely - necessary to be effective following a config change, as status - message may not instantly update to non-ready.""" - - if init_sleep: - time.sleep(init_sleep) - - message = re.compile('^Unit is ready and clustered$') - deployment._auto_wait_for_status(message=message, - timeout=timeout, - include_only=['rabbitmq-server']) - - def add_rmq_test_user(self, sentry_units, - username="testuser1", password="changeme"): - """Add a test user via the first rmq juju unit, check connection as - the new user against all sentry units. - - :param sentry_units: list of sentry unit pointers - :param username: amqp user name, default to testuser1 - :param password: amqp user password - :returns: None if successful. Raise on error. - """ - self.log.debug('Adding rmq user ({})...'.format(username)) - - # Check that user does not already exist - cmd_user_list = 'rabbitmqctl list_users' - output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_list) - if username in output: - self.log.warning('User ({}) already exists, returning ' - 'gracefully.'.format(username)) - return - - perms = '".*" ".*" ".*"' - cmds = ['rabbitmqctl add_user {} {}'.format(username, password), - 'rabbitmqctl set_permissions {} {}'.format(username, perms)] - - # Add user via first unit - for cmd in cmds: - output, _ = self.run_cmd_unit(sentry_units[0], cmd) - - # Check connection against the other sentry_units - self.log.debug('Checking user connect against units...') - for sentry_unit in sentry_units: - connection = self.connect_amqp_by_unit(sentry_unit, ssl=False, - username=username, - password=password) - connection.close() - - def delete_rmq_test_user(self, sentry_units, username="testuser1"): - """Delete a rabbitmq user via the first rmq juju unit. - - :param sentry_units: list of sentry unit pointers - :param username: amqp user name, default to testuser1 - :param password: amqp user password - :returns: None if successful or no such user. - """ - self.log.debug('Deleting rmq user ({})...'.format(username)) - - # Check that the user exists - cmd_user_list = 'rabbitmqctl list_users' - output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_list) - - if username not in output: - self.log.warning('User ({}) does not exist, returning ' - 'gracefully.'.format(username)) - return - - # Delete the user - cmd_user_del = 'rabbitmqctl delete_user {}'.format(username) - output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_del) - - def get_rmq_cluster_status(self, sentry_unit): - """Execute rabbitmq cluster status command on a unit and return - the full output. - - :param unit: sentry unit - :returns: String containing console output of cluster status command - """ - cmd = 'rabbitmqctl cluster_status' - output, _ = self.run_cmd_unit(sentry_unit, cmd) - self.log.debug('{} cluster_status:\n{}'.format( - sentry_unit.info['unit_name'], output)) - return str(output) - - def get_rmq_cluster_running_nodes(self, sentry_unit): - """Parse rabbitmqctl cluster_status output string, return list of - running rabbitmq cluster nodes. - - :param unit: sentry unit - :returns: List containing node names of running nodes - """ - # NOTE(beisner): rabbitmqctl cluster_status output is not - # json-parsable, do string chop foo, then json.loads that. - str_stat = self.get_rmq_cluster_status(sentry_unit) - if 'running_nodes' in str_stat: - pos_start = str_stat.find("{running_nodes,") + 15 - pos_end = str_stat.find("]},", pos_start) + 1 - str_run_nodes = str_stat[pos_start:pos_end].replace("'", '"') - run_nodes = json.loads(str_run_nodes) - return run_nodes - else: - return [] - - def validate_rmq_cluster_running_nodes(self, sentry_units): - """Check that all rmq unit hostnames are represented in the - cluster_status output of all units. - - :param host_names: dict of juju unit names to host names - :param units: list of sentry unit pointers (all rmq units) - :returns: None if successful, otherwise return error message - """ - host_names = self.get_unit_hostnames(sentry_units) - errors = [] - - # Query every unit for cluster_status running nodes - for query_unit in sentry_units: - query_unit_name = query_unit.info['unit_name'] - running_nodes = self.get_rmq_cluster_running_nodes(query_unit) - - # Confirm that every unit is represented in the queried unit's - # cluster_status running nodes output. - for validate_unit in sentry_units: - val_host_name = host_names[validate_unit.info['unit_name']] - val_node_name = 'rabbit@{}'.format(val_host_name) - - if val_node_name not in running_nodes: - errors.append('Cluster member check failed on {}: {} not ' - 'in {}\n'.format(query_unit_name, - val_node_name, - running_nodes)) - if errors: - return ''.join(errors) - - def rmq_ssl_is_enabled_on_unit(self, sentry_unit, port=None): - """Check a single juju rmq unit for ssl and port in the config file.""" - host = sentry_unit.info['public-address'] - unit_name = sentry_unit.info['unit_name'] - - conf_file = '/etc/rabbitmq/rabbitmq.config' - conf_contents = str(self.file_contents_safe(sentry_unit, - conf_file, max_wait=16)) - # Checks - conf_ssl = 'ssl' in conf_contents - conf_port = str(port) in conf_contents - - # Port explicitly checked in config - if port and conf_port and conf_ssl: - self.log.debug('SSL is enabled @{}:{} ' - '({})'.format(host, port, unit_name)) - return True - elif port and not conf_port and conf_ssl: - self.log.debug('SSL is enabled @{} but not on port {} ' - '({})'.format(host, port, unit_name)) - return False - # Port not checked (useful when checking that ssl is disabled) - elif not port and conf_ssl: - self.log.debug('SSL is enabled @{}:{} ' - '({})'.format(host, port, unit_name)) - return True - elif not conf_ssl: - self.log.debug('SSL not enabled @{}:{} ' - '({})'.format(host, port, unit_name)) - return False - else: - msg = ('Unknown condition when checking SSL status @{}:{} ' - '({})'.format(host, port, unit_name)) - amulet.raise_status(amulet.FAIL, msg) - - def validate_rmq_ssl_enabled_units(self, sentry_units, port=None): - """Check that ssl is enabled on rmq juju sentry units. - - :param sentry_units: list of all rmq sentry units - :param port: optional ssl port override to validate - :returns: None if successful, otherwise return error message - """ - for sentry_unit in sentry_units: - if not self.rmq_ssl_is_enabled_on_unit(sentry_unit, port=port): - return ('Unexpected condition: ssl is disabled on unit ' - '({})'.format(sentry_unit.info['unit_name'])) - return None - - def validate_rmq_ssl_disabled_units(self, sentry_units): - """Check that ssl is enabled on listed rmq juju sentry units. - - :param sentry_units: list of all rmq sentry units - :returns: True if successful. Raise on error. - """ - for sentry_unit in sentry_units: - if self.rmq_ssl_is_enabled_on_unit(sentry_unit): - return ('Unexpected condition: ssl is enabled on unit ' - '({})'.format(sentry_unit.info['unit_name'])) - return None - - def configure_rmq_ssl_on(self, sentry_units, deployment, - port=None, max_wait=60): - """Turn ssl charm config option on, with optional non-default - ssl port specification. Confirm that it is enabled on every - unit. - - :param sentry_units: list of sentry units - :param deployment: amulet deployment object pointer - :param port: amqp port, use defaults if None - :param max_wait: maximum time to wait in seconds to confirm - :returns: None if successful. Raise on error. - """ - self.log.debug('Setting ssl charm config option: on') - - # Enable RMQ SSL - config = {'ssl': 'on'} - if port: - config['ssl_port'] = port - - deployment.d.configure('rabbitmq-server', config) - - # Wait for unit status - self.rmq_wait_for_cluster(deployment) - - # Confirm - tries = 0 - ret = self.validate_rmq_ssl_enabled_units(sentry_units, port=port) - while ret and tries < (max_wait / 4): - time.sleep(4) - self.log.debug('Attempt {}: {}'.format(tries, ret)) - ret = self.validate_rmq_ssl_enabled_units(sentry_units, port=port) - tries += 1 - - if ret: - amulet.raise_status(amulet.FAIL, ret) - - def configure_rmq_ssl_off(self, sentry_units, deployment, max_wait=60): - """Turn ssl charm config option off, confirm that it is disabled - on every unit. - - :param sentry_units: list of sentry units - :param deployment: amulet deployment object pointer - :param max_wait: maximum time to wait in seconds to confirm - :returns: None if successful. Raise on error. - """ - self.log.debug('Setting ssl charm config option: off') - - # Disable RMQ SSL - config = {'ssl': 'off'} - deployment.d.configure('rabbitmq-server', config) - - # Wait for unit status - self.rmq_wait_for_cluster(deployment) - - # Confirm - tries = 0 - ret = self.validate_rmq_ssl_disabled_units(sentry_units) - while ret and tries < (max_wait / 4): - time.sleep(4) - self.log.debug('Attempt {}: {}'.format(tries, ret)) - ret = self.validate_rmq_ssl_disabled_units(sentry_units) - tries += 1 - - if ret: - amulet.raise_status(amulet.FAIL, ret) - - def connect_amqp_by_unit(self, sentry_unit, ssl=False, - port=None, fatal=True, - username="testuser1", password="changeme"): - """Establish and return a pika amqp connection to the rabbitmq service - running on a rmq juju unit. - - :param sentry_unit: sentry unit pointer - :param ssl: boolean, default to False - :param port: amqp port, use defaults if None - :param fatal: boolean, default to True (raises on connect error) - :param username: amqp user name, default to testuser1 - :param password: amqp user password - :returns: pika amqp connection pointer or None if failed and non-fatal - """ - host = sentry_unit.info['public-address'] - unit_name = sentry_unit.info['unit_name'] - - # Default port logic if port is not specified - if ssl and not port: - port = 5671 - elif not ssl and not port: - port = 5672 - - self.log.debug('Connecting to amqp on {}:{} ({}) as ' - '{}...'.format(host, port, unit_name, username)) - - try: - credentials = pika.PlainCredentials(username, password) - parameters = pika.ConnectionParameters(host=host, port=port, - credentials=credentials, - ssl=ssl, - connection_attempts=3, - retry_delay=5, - socket_timeout=1) - connection = pika.BlockingConnection(parameters) - assert connection.is_open is True - assert connection.is_closing is False - self.log.debug('Connect OK') - return connection - except Exception as e: - msg = ('amqp connection failed to {}:{} as ' - '{} ({})'.format(host, port, username, str(e))) - if fatal: - amulet.raise_status(amulet.FAIL, msg) - else: - self.log.warn(msg) - return None - - def publish_amqp_message_by_unit(self, sentry_unit, message, - queue="test", ssl=False, - username="testuser1", - password="changeme", - port=None): - """Publish an amqp message to a rmq juju unit. - - :param sentry_unit: sentry unit pointer - :param message: amqp message string - :param queue: message queue, default to test - :param username: amqp user name, default to testuser1 - :param password: amqp user password - :param ssl: boolean, default to False - :param port: amqp port, use defaults if None - :returns: None. Raises exception if publish failed. - """ - self.log.debug('Publishing message to {} queue:\n{}'.format(queue, - message)) - connection = self.connect_amqp_by_unit(sentry_unit, ssl=ssl, - port=port, - username=username, - password=password) - - # NOTE(beisner): extra debug here re: pika hang potential: - # https://github.com/pika/pika/issues/297 - # https://groups.google.com/forum/#!topic/rabbitmq-users/Ja0iyfF0Szw - self.log.debug('Defining channel...') - channel = connection.channel() - self.log.debug('Declaring queue...') - channel.queue_declare(queue=queue, auto_delete=False, durable=True) - self.log.debug('Publishing message...') - channel.basic_publish(exchange='', routing_key=queue, body=message) - self.log.debug('Closing channel...') - channel.close() - self.log.debug('Closing connection...') - connection.close() - - def get_amqp_message_by_unit(self, sentry_unit, queue="test", - username="testuser1", - password="changeme", - ssl=False, port=None): - """Get an amqp message from a rmq juju unit. - - :param sentry_unit: sentry unit pointer - :param queue: message queue, default to test - :param username: amqp user name, default to testuser1 - :param password: amqp user password - :param ssl: boolean, default to False - :param port: amqp port, use defaults if None - :returns: amqp message body as string. Raise if get fails. - """ - connection = self.connect_amqp_by_unit(sentry_unit, ssl=ssl, - port=port, - username=username, - password=password) - channel = connection.channel() - method_frame, _, body = channel.basic_get(queue) - - if method_frame: - self.log.debug('Retreived message from {} queue:\n{}'.format(queue, - body)) - channel.basic_ack(method_frame.delivery_tag) - channel.close() - connection.close() - return body - else: - msg = 'No message retrieved.' - amulet.raise_status(amulet.FAIL, msg) - - def validate_memcache(self, sentry_unit, conf, os_release, - earliest_release=5, section='keystone_authtoken', - check_kvs=None): - """Check Memcache is running and is configured to be used - - Example call from Amulet test: - - def test_110_memcache(self): - u.validate_memcache(self.neutron_api_sentry, - '/etc/neutron/neutron.conf', - self._get_openstack_release()) - - :param sentry_unit: sentry unit - :param conf: OpenStack config file to check memcache settings - :param os_release: Current OpenStack release int code - :param earliest_release: Earliest Openstack release to check int code - :param section: OpenStack config file section to check - :param check_kvs: Dict of settings to check in config file - :returns: None - """ - if os_release < earliest_release: - self.log.debug('Skipping memcache checks for deployment. {} <' - 'mitaka'.format(os_release)) - return - _kvs = check_kvs or {'memcached_servers': 'inet6:[::1]:11211'} - self.log.debug('Checking memcached is running') - ret = self.validate_services_by_name({sentry_unit: ['memcached']}) - if ret: - amulet.raise_status(amulet.FAIL, msg='Memcache running check' - 'failed {}'.format(ret)) - else: - self.log.debug('OK') - self.log.debug('Checking memcache url is configured in {}'.format( - conf)) - if self.validate_config_data(sentry_unit, conf, section, _kvs): - message = "Memcache config error in: {}".format(conf) - amulet.raise_status(amulet.FAIL, msg=message) - else: - self.log.debug('OK') - self.log.debug('Checking memcache configuration in ' - '/etc/memcached.conf') - contents = self.file_contents_safe(sentry_unit, '/etc/memcached.conf', - fatal=True) - ubuntu_release, _ = self.run_cmd_unit(sentry_unit, 'lsb_release -cs') - if CompareHostReleases(ubuntu_release) <= 'trusty': - memcache_listen_addr = 'ip6-localhost' - else: - memcache_listen_addr = '::1' - expected = { - '-p': '11211', - '-l': memcache_listen_addr} - found = [] - for key, value in expected.items(): - for line in contents.split('\n'): - if line.startswith(key): - self.log.debug('Checking {} is set to {}'.format( - key, - value)) - assert value == line.split()[-1] - self.log.debug(line.split()[-1]) - found.append(key) - if sorted(found) == sorted(expected.keys()): - self.log.debug('OK') - else: - message = "Memcache config error in: /etc/memcached.conf" - amulet.raise_status(amulet.FAIL, msg=message) diff --git a/tests/charmhelpers/core/__init__.py b/tests/charmhelpers/core/__init__.py deleted file mode 100644 index d7567b8..0000000 --- a/tests/charmhelpers/core/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tests/charmhelpers/core/decorators.py b/tests/charmhelpers/core/decorators.py deleted file mode 100644 index 6ad41ee..0000000 --- a/tests/charmhelpers/core/decorators.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# -# Copyright 2014 Canonical Ltd. -# -# Authors: -# Edward Hope-Morley -# - -import time - -from charmhelpers.core.hookenv import ( - log, - INFO, -) - - -def retry_on_exception(num_retries, base_delay=0, exc_type=Exception): - """If the decorated function raises exception exc_type, allow num_retries - retry attempts before raise the exception. - """ - def _retry_on_exception_inner_1(f): - def _retry_on_exception_inner_2(*args, **kwargs): - retries = num_retries - multiplier = 1 - while True: - try: - return f(*args, **kwargs) - except exc_type: - if not retries: - raise - - delay = base_delay * multiplier - multiplier += 1 - log("Retrying '%s' %d more times (delay=%s)" % - (f.__name__, retries, delay), level=INFO) - retries -= 1 - if delay: - time.sleep(delay) - - return _retry_on_exception_inner_2 - - return _retry_on_exception_inner_1 diff --git a/tests/charmhelpers/core/files.py b/tests/charmhelpers/core/files.py deleted file mode 100644 index fdd82b7..0000000 --- a/tests/charmhelpers/core/files.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__author__ = 'Jorge Niedbalski ' - -import os -import subprocess - - -def sed(filename, before, after, flags='g'): - """ - Search and replaces the given pattern on filename. - - :param filename: relative or absolute file path. - :param before: expression to be replaced (see 'man sed') - :param after: expression to replace with (see 'man sed') - :param flags: sed-compatible regex flags in example, to make - the search and replace case insensitive, specify ``flags="i"``. - The ``g`` flag is always specified regardless, so you do not - need to remember to include it when overriding this parameter. - :returns: If the sed command exit code was zero then return, - otherwise raise CalledProcessError. - """ - expression = r's/{0}/{1}/{2}'.format(before, - after, flags) - - return subprocess.check_call(["sed", "-i", "-r", "-e", - expression, - os.path.expanduser(filename)]) diff --git a/tests/charmhelpers/core/fstab.py b/tests/charmhelpers/core/fstab.py deleted file mode 100644 index d9fa915..0000000 --- a/tests/charmhelpers/core/fstab.py +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import io -import os - -__author__ = 'Jorge Niedbalski R. ' - - -class Fstab(io.FileIO): - """This class extends file in order to implement a file reader/writer - for file `/etc/fstab` - """ - - class Entry(object): - """Entry class represents a non-comment line on the `/etc/fstab` file - """ - def __init__(self, device, mountpoint, filesystem, - options, d=0, p=0): - self.device = device - self.mountpoint = mountpoint - self.filesystem = filesystem - - if not options: - options = "defaults" - - self.options = options - self.d = int(d) - self.p = int(p) - - def __eq__(self, o): - return str(self) == str(o) - - def __str__(self): - return "{} {} {} {} {} {}".format(self.device, - self.mountpoint, - self.filesystem, - self.options, - self.d, - self.p) - - DEFAULT_PATH = os.path.join(os.path.sep, 'etc', 'fstab') - - def __init__(self, path=None): - if path: - self._path = path - else: - self._path = self.DEFAULT_PATH - super(Fstab, self).__init__(self._path, 'rb+') - - def _hydrate_entry(self, line): - # NOTE: use split with no arguments to split on any - # whitespace including tabs - return Fstab.Entry(*filter( - lambda x: x not in ('', None), - line.strip("\n").split())) - - @property - def entries(self): - self.seek(0) - for line in self.readlines(): - line = line.decode('us-ascii') - try: - if line.strip() and not line.strip().startswith("#"): - yield self._hydrate_entry(line) - except ValueError: - pass - - def get_entry_by_attr(self, attr, value): - for entry in self.entries: - e_attr = getattr(entry, attr) - if e_attr == value: - return entry - return None - - def add_entry(self, entry): - if self.get_entry_by_attr('device', entry.device): - return False - - self.write((str(entry) + '\n').encode('us-ascii')) - self.truncate() - return entry - - def remove_entry(self, entry): - self.seek(0) - - lines = [l.decode('us-ascii') for l in self.readlines()] - - found = False - for index, line in enumerate(lines): - if line.strip() and not line.strip().startswith("#"): - if self._hydrate_entry(line) == entry: - found = True - break - - if not found: - return False - - lines.remove(line) - - self.seek(0) - self.write(''.join(lines).encode('us-ascii')) - self.truncate() - return True - - @classmethod - def remove_by_mountpoint(cls, mountpoint, path=None): - fstab = cls(path=path) - entry = fstab.get_entry_by_attr('mountpoint', mountpoint) - if entry: - return fstab.remove_entry(entry) - return False - - @classmethod - def add(cls, device, mountpoint, filesystem, options=None, path=None): - return cls(path=path).add_entry(Fstab.Entry(device, - mountpoint, filesystem, - options=options)) diff --git a/tests/charmhelpers/core/hookenv.py b/tests/charmhelpers/core/hookenv.py deleted file mode 100644 index 5a88f79..0000000 --- a/tests/charmhelpers/core/hookenv.py +++ /dev/null @@ -1,1206 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"Interactions with the Juju environment" -# Copyright 2013 Canonical Ltd. -# -# Authors: -# Charm Helpers Developers - -from __future__ import print_function -import copy -from distutils.version import LooseVersion -from functools import wraps -from collections import namedtuple -import glob -import os -import json -import yaml -import subprocess -import sys -import errno -import tempfile -from subprocess import CalledProcessError - -import six -if not six.PY3: - from UserDict import UserDict -else: - from collections import UserDict - -CRITICAL = "CRITICAL" -ERROR = "ERROR" -WARNING = "WARNING" -INFO = "INFO" -DEBUG = "DEBUG" -TRACE = "TRACE" -MARKER = object() - -cache = {} - - -def cached(func): - """Cache return values for multiple executions of func + args - - For example:: - - @cached - def unit_get(attribute): - pass - - unit_get('test') - - will cache the result of unit_get + 'test' for future calls. - """ - @wraps(func) - def wrapper(*args, **kwargs): - global cache - key = str((func, args, kwargs)) - try: - return cache[key] - except KeyError: - pass # Drop out of the exception handler scope. - res = func(*args, **kwargs) - cache[key] = res - return res - wrapper._wrapped = func - return wrapper - - -def flush(key): - """Flushes any entries from function cache where the - key is found in the function+args """ - flush_list = [] - for item in cache: - if key in item: - flush_list.append(item) - for item in flush_list: - del cache[item] - - -def log(message, level=None): - """Write a message to the juju log""" - command = ['juju-log'] - if level: - command += ['-l', level] - if not isinstance(message, six.string_types): - message = repr(message) - command += [message] - # Missing juju-log should not cause failures in unit tests - # Send log output to stderr - try: - subprocess.call(command) - except OSError as e: - if e.errno == errno.ENOENT: - if level: - message = "{}: {}".format(level, message) - message = "juju-log: {}".format(message) - print(message, file=sys.stderr) - else: - raise - - -class Serializable(UserDict): - """Wrapper, an object that can be serialized to yaml or json""" - - def __init__(self, obj): - # wrap the object - UserDict.__init__(self) - self.data = obj - - def __getattr__(self, attr): - # See if this object has attribute. - if attr in ("json", "yaml", "data"): - return self.__dict__[attr] - # Check for attribute in wrapped object. - got = getattr(self.data, attr, MARKER) - if got is not MARKER: - return got - # Proxy to the wrapped object via dict interface. - try: - return self.data[attr] - except KeyError: - raise AttributeError(attr) - - def __getstate__(self): - # Pickle as a standard dictionary. - return self.data - - def __setstate__(self, state): - # Unpickle into our wrapper. - self.data = state - - def json(self): - """Serialize the object to json""" - return json.dumps(self.data) - - def yaml(self): - """Serialize the object to yaml""" - return yaml.dump(self.data) - - -def execution_environment(): - """A convenient bundling of the current execution context""" - context = {} - context['conf'] = config() - if relation_id(): - context['reltype'] = relation_type() - context['relid'] = relation_id() - context['rel'] = relation_get() - context['unit'] = local_unit() - context['rels'] = relations() - context['env'] = os.environ - return context - - -def in_relation_hook(): - """Determine whether we're running in a relation hook""" - return 'JUJU_RELATION' in os.environ - - -def relation_type(): - """The scope for the current relation hook""" - return os.environ.get('JUJU_RELATION', None) - - -@cached -def relation_id(relation_name=None, service_or_unit=None): - """The relation ID for the current or a specified relation""" - if not relation_name and not service_or_unit: - return os.environ.get('JUJU_RELATION_ID', None) - elif relation_name and service_or_unit: - service_name = service_or_unit.split('/')[0] - for relid in relation_ids(relation_name): - remote_service = remote_service_name(relid) - if remote_service == service_name: - return relid - else: - raise ValueError('Must specify neither or both of relation_name and service_or_unit') - - -def local_unit(): - """Local unit ID""" - return os.environ['JUJU_UNIT_NAME'] - - -def remote_unit(): - """The remote unit for the current relation hook""" - return os.environ.get('JUJU_REMOTE_UNIT', None) - - -def service_name(): - """The name service group this unit belongs to""" - return local_unit().split('/')[0] - - -def principal_unit(): - """Returns the principal unit of this unit, otherwise None""" - # Juju 2.2 and above provides JUJU_PRINCIPAL_UNIT - principal_unit = os.environ.get('JUJU_PRINCIPAL_UNIT', None) - # If it's empty, then this unit is the principal - if principal_unit == '': - return os.environ['JUJU_UNIT_NAME'] - elif principal_unit is not None: - return principal_unit - # For Juju 2.1 and below, let's try work out the principle unit by - # the various charms' metadata.yaml. - for reltype in relation_types(): - for rid in relation_ids(reltype): - for unit in related_units(rid): - md = _metadata_unit(unit) - if not md: - continue - subordinate = md.pop('subordinate', None) - if not subordinate: - return unit - return None - - -@cached -def remote_service_name(relid=None): - """The remote service name for a given relation-id (or the current relation)""" - if relid is None: - unit = remote_unit() - else: - units = related_units(relid) - unit = units[0] if units else None - return unit.split('/')[0] if unit else None - - -def hook_name(): - """The name of the currently executing hook""" - return os.environ.get('JUJU_HOOK_NAME', os.path.basename(sys.argv[0])) - - -class Config(dict): - """A dictionary representation of the charm's config.yaml, with some - extra features: - - - See which values in the dictionary have changed since the previous hook. - - For values that have changed, see what the previous value was. - - Store arbitrary data for use in a later hook. - - NOTE: Do not instantiate this object directly - instead call - ``hookenv.config()``, which will return an instance of :class:`Config`. - - Example usage:: - - >>> # inside a hook - >>> from charmhelpers.core import hookenv - >>> config = hookenv.config() - >>> config['foo'] - 'bar' - >>> # store a new key/value for later use - >>> config['mykey'] = 'myval' - - - >>> # user runs `juju set mycharm foo=baz` - >>> # now we're inside subsequent config-changed hook - >>> config = hookenv.config() - >>> config['foo'] - 'baz' - >>> # test to see if this val has changed since last hook - >>> config.changed('foo') - True - >>> # what was the previous value? - >>> config.previous('foo') - 'bar' - >>> # keys/values that we add are preserved across hooks - >>> config['mykey'] - 'myval' - - """ - CONFIG_FILE_NAME = '.juju-persistent-config' - - def __init__(self, *args, **kw): - super(Config, self).__init__(*args, **kw) - self.implicit_save = True - self._prev_dict = None - self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME) - if os.path.exists(self.path): - self.load_previous() - atexit(self._implicit_save) - - def load_previous(self, path=None): - """Load previous copy of config from disk. - - In normal usage you don't need to call this method directly - it - is called automatically at object initialization. - - :param path: - - File path from which to load the previous config. If `None`, - config is loaded from the default location. If `path` is - specified, subsequent `save()` calls will write to the same - path. - - """ - self.path = path or self.path - with open(self.path) as f: - self._prev_dict = json.load(f) - for k, v in copy.deepcopy(self._prev_dict).items(): - if k not in self: - self[k] = v - - def changed(self, key): - """Return True if the current value for this key is different from - the previous value. - - """ - if self._prev_dict is None: - return True - return self.previous(key) != self.get(key) - - def previous(self, key): - """Return previous value for this key, or None if there - is no previous value. - - """ - if self._prev_dict: - return self._prev_dict.get(key) - return None - - def save(self): - """Save this config to disk. - - If the charm is using the :mod:`Services Framework ` - or :meth:'@hook ' decorator, this - is called automatically at the end of successful hook execution. - Otherwise, it should be called directly by user code. - - To disable automatic saves, set ``implicit_save=False`` on this - instance. - - """ - with open(self.path, 'w') as f: - json.dump(self, f) - - def _implicit_save(self): - if self.implicit_save: - self.save() - - -@cached -def config(scope=None): - """Juju charm configuration""" - config_cmd_line = ['config-get'] - if scope is not None: - config_cmd_line.append(scope) - else: - config_cmd_line.append('--all') - config_cmd_line.append('--format=json') - try: - config_data = json.loads( - subprocess.check_output(config_cmd_line).decode('UTF-8')) - if scope is not None: - return config_data - return Config(config_data) - except ValueError: - return None - - -@cached -def relation_get(attribute=None, unit=None, rid=None): - """Get relation information""" - _args = ['relation-get', '--format=json'] - if rid: - _args.append('-r') - _args.append(rid) - _args.append(attribute or '-') - if unit: - _args.append(unit) - try: - return json.loads(subprocess.check_output(_args).decode('UTF-8')) - except ValueError: - return None - except CalledProcessError as e: - if e.returncode == 2: - return None - raise - - -def relation_set(relation_id=None, relation_settings=None, **kwargs): - """Set relation information for the current unit""" - relation_settings = relation_settings if relation_settings else {} - relation_cmd_line = ['relation-set'] - accepts_file = "--file" in subprocess.check_output( - relation_cmd_line + ["--help"], universal_newlines=True) - if relation_id is not None: - relation_cmd_line.extend(('-r', relation_id)) - settings = relation_settings.copy() - settings.update(kwargs) - for key, value in settings.items(): - # Force value to be a string: it always should, but some call - # sites pass in things like dicts or numbers. - if value is not None: - settings[key] = "{}".format(value) - if accepts_file: - # --file was introduced in Juju 1.23.2. Use it by default if - # available, since otherwise we'll break if the relation data is - # too big. Ideally we should tell relation-set to read the data from - # stdin, but that feature is broken in 1.23.2: Bug #1454678. - with tempfile.NamedTemporaryFile(delete=False) as settings_file: - settings_file.write(yaml.safe_dump(settings).encode("utf-8")) - subprocess.check_call( - relation_cmd_line + ["--file", settings_file.name]) - os.remove(settings_file.name) - else: - for key, value in settings.items(): - if value is None: - relation_cmd_line.append('{}='.format(key)) - else: - relation_cmd_line.append('{}={}'.format(key, value)) - subprocess.check_call(relation_cmd_line) - # Flush cache of any relation-gets for local unit - flush(local_unit()) - - -def relation_clear(r_id=None): - ''' Clears any relation data already set on relation r_id ''' - settings = relation_get(rid=r_id, - unit=local_unit()) - for setting in settings: - if setting not in ['public-address', 'private-address']: - settings[setting] = None - relation_set(relation_id=r_id, - **settings) - - -@cached -def relation_ids(reltype=None): - """A list of relation_ids""" - reltype = reltype or relation_type() - relid_cmd_line = ['relation-ids', '--format=json'] - if reltype is not None: - relid_cmd_line.append(reltype) - return json.loads( - subprocess.check_output(relid_cmd_line).decode('UTF-8')) or [] - return [] - - -@cached -def related_units(relid=None): - """A list of related units""" - relid = relid or relation_id() - units_cmd_line = ['relation-list', '--format=json'] - if relid is not None: - units_cmd_line.extend(('-r', relid)) - return json.loads( - subprocess.check_output(units_cmd_line).decode('UTF-8')) or [] - - -@cached -def relation_for_unit(unit=None, rid=None): - """Get the json represenation of a unit's relation""" - unit = unit or remote_unit() - relation = relation_get(unit=unit, rid=rid) - for key in relation: - if key.endswith('-list'): - relation[key] = relation[key].split() - relation['__unit__'] = unit - return relation - - -@cached -def relations_for_id(relid=None): - """Get relations of a specific relation ID""" - relation_data = [] - relid = relid or relation_ids() - for unit in related_units(relid): - unit_data = relation_for_unit(unit, relid) - unit_data['__relid__'] = relid - relation_data.append(unit_data) - return relation_data - - -@cached -def relations_of_type(reltype=None): - """Get relations of a specific type""" - relation_data = [] - reltype = reltype or relation_type() - for relid in relation_ids(reltype): - for relation in relations_for_id(relid): - relation['__relid__'] = relid - relation_data.append(relation) - return relation_data - - -@cached -def metadata(): - """Get the current charm metadata.yaml contents as a python object""" - with open(os.path.join(charm_dir(), 'metadata.yaml')) as md: - return yaml.safe_load(md) - - -def _metadata_unit(unit): - """Given the name of a unit (e.g. apache2/0), get the unit charm's - metadata.yaml. Very similar to metadata() but allows us to inspect - other units. Unit needs to be co-located, such as a subordinate or - principal/primary. - - :returns: metadata.yaml as a python object. - - """ - basedir = os.sep.join(charm_dir().split(os.sep)[:-2]) - unitdir = 'unit-{}'.format(unit.replace(os.sep, '-')) - joineddir = os.path.join(basedir, unitdir, 'charm', 'metadata.yaml') - if not os.path.exists(joineddir): - return None - with open(joineddir) as md: - return yaml.safe_load(md) - - -@cached -def relation_types(): - """Get a list of relation types supported by this charm""" - rel_types = [] - md = metadata() - for key in ('provides', 'requires', 'peers'): - section = md.get(key) - if section: - rel_types.extend(section.keys()) - return rel_types - - -@cached -def peer_relation_id(): - '''Get the peers relation id if a peers relation has been joined, else None.''' - md = metadata() - section = md.get('peers') - if section: - for key in section: - relids = relation_ids(key) - if relids: - return relids[0] - return None - - -@cached -def relation_to_interface(relation_name): - """ - Given the name of a relation, return the interface that relation uses. - - :returns: The interface name, or ``None``. - """ - return relation_to_role_and_interface(relation_name)[1] - - -@cached -def relation_to_role_and_interface(relation_name): - """ - Given the name of a relation, return the role and the name of the interface - that relation uses (where role is one of ``provides``, ``requires``, or ``peers``). - - :returns: A tuple containing ``(role, interface)``, or ``(None, None)``. - """ - _metadata = metadata() - for role in ('provides', 'requires', 'peers'): - interface = _metadata.get(role, {}).get(relation_name, {}).get('interface') - if interface: - return role, interface - return None, None - - -@cached -def role_and_interface_to_relations(role, interface_name): - """ - Given a role and interface name, return a list of relation names for the - current charm that use that interface under that role (where role is one - of ``provides``, ``requires``, or ``peers``). - - :returns: A list of relation names. - """ - _metadata = metadata() - results = [] - for relation_name, relation in _metadata.get(role, {}).items(): - if relation['interface'] == interface_name: - results.append(relation_name) - return results - - -@cached -def interface_to_relations(interface_name): - """ - Given an interface, return a list of relation names for the current - charm that use that interface. - - :returns: A list of relation names. - """ - results = [] - for role in ('provides', 'requires', 'peers'): - results.extend(role_and_interface_to_relations(role, interface_name)) - return results - - -@cached -def charm_name(): - """Get the name of the current charm as is specified on metadata.yaml""" - return metadata().get('name') - - -@cached -def relations(): - """Get a nested dictionary of relation data for all related units""" - rels = {} - for reltype in relation_types(): - relids = {} - for relid in relation_ids(reltype): - units = {local_unit(): relation_get(unit=local_unit(), rid=relid)} - for unit in related_units(relid): - reldata = relation_get(unit=unit, rid=relid) - units[unit] = reldata - relids[relid] = units - rels[reltype] = relids - return rels - - -@cached -def is_relation_made(relation, keys='private-address'): - ''' - Determine whether a relation is established by checking for - presence of key(s). If a list of keys is provided, they - must all be present for the relation to be identified as made - ''' - if isinstance(keys, str): - keys = [keys] - for r_id in relation_ids(relation): - for unit in related_units(r_id): - context = {} - for k in keys: - context[k] = relation_get(k, rid=r_id, - unit=unit) - if None not in context.values(): - return True - return False - - -def _port_op(op_name, port, protocol="TCP"): - """Open or close a service network port""" - _args = [op_name] - icmp = protocol.upper() == "ICMP" - if icmp: - _args.append(protocol) - else: - _args.append('{}/{}'.format(port, protocol)) - try: - subprocess.check_call(_args) - except subprocess.CalledProcessError: - # Older Juju pre 2.3 doesn't support ICMP - # so treat it as a no-op if it fails. - if not icmp: - raise - - -def open_port(port, protocol="TCP"): - """Open a service network port""" - _port_op('open-port', port, protocol) - - -def close_port(port, protocol="TCP"): - """Close a service network port""" - _port_op('close-port', port, protocol) - - -def open_ports(start, end, protocol="TCP"): - """Opens a range of service network ports""" - _args = ['open-port'] - _args.append('{}-{}/{}'.format(start, end, protocol)) - subprocess.check_call(_args) - - -def close_ports(start, end, protocol="TCP"): - """Close a range of service network ports""" - _args = ['close-port'] - _args.append('{}-{}/{}'.format(start, end, protocol)) - subprocess.check_call(_args) - - -def opened_ports(): - """Get the opened ports - - *Note that this will only show ports opened in a previous hook* - - :returns: Opened ports as a list of strings: ``['8080/tcp', '8081-8083/tcp']`` - """ - _args = ['opened-ports', '--format=json'] - return json.loads(subprocess.check_output(_args).decode('UTF-8')) - - -@cached -def unit_get(attribute): - """Get the unit ID for the remote unit""" - _args = ['unit-get', '--format=json', attribute] - try: - return json.loads(subprocess.check_output(_args).decode('UTF-8')) - except ValueError: - return None - - -def unit_public_ip(): - """Get this unit's public IP address""" - return unit_get('public-address') - - -def unit_private_ip(): - """Get this unit's private IP address""" - return unit_get('private-address') - - -@cached -def storage_get(attribute=None, storage_id=None): - """Get storage attributes""" - _args = ['storage-get', '--format=json'] - if storage_id: - _args.extend(('-s', storage_id)) - if attribute: - _args.append(attribute) - try: - return json.loads(subprocess.check_output(_args).decode('UTF-8')) - except ValueError: - return None - - -@cached -def storage_list(storage_name=None): - """List the storage IDs for the unit""" - _args = ['storage-list', '--format=json'] - if storage_name: - _args.append(storage_name) - try: - return json.loads(subprocess.check_output(_args).decode('UTF-8')) - except ValueError: - return None - except OSError as e: - import errno - if e.errno == errno.ENOENT: - # storage-list does not exist - return [] - raise - - -class UnregisteredHookError(Exception): - """Raised when an undefined hook is called""" - pass - - -class Hooks(object): - """A convenient handler for hook functions. - - Example:: - - hooks = Hooks() - - # register a hook, taking its name from the function name - @hooks.hook() - def install(): - pass # your code here - - # register a hook, providing a custom hook name - @hooks.hook("config-changed") - def config_changed(): - pass # your code here - - if __name__ == "__main__": - # execute a hook based on the name the program is called by - hooks.execute(sys.argv) - """ - - def __init__(self, config_save=None): - super(Hooks, self).__init__() - self._hooks = {} - - # For unknown reasons, we allow the Hooks constructor to override - # config().implicit_save. - if config_save is not None: - config().implicit_save = config_save - - def register(self, name, function): - """Register a hook""" - self._hooks[name] = function - - def execute(self, args): - """Execute a registered hook based on args[0]""" - _run_atstart() - hook_name = os.path.basename(args[0]) - if hook_name in self._hooks: - try: - self._hooks[hook_name]() - except SystemExit as x: - if x.code is None or x.code == 0: - _run_atexit() - raise - _run_atexit() - else: - raise UnregisteredHookError(hook_name) - - def hook(self, *hook_names): - """Decorator, registering them as hooks""" - def wrapper(decorated): - for hook_name in hook_names: - self.register(hook_name, decorated) - else: - self.register(decorated.__name__, decorated) - if '_' in decorated.__name__: - self.register( - decorated.__name__.replace('_', '-'), decorated) - return decorated - return wrapper - - -def charm_dir(): - """Return the root directory of the current charm""" - d = os.environ.get('JUJU_CHARM_DIR') - if d is not None: - return d - return os.environ.get('CHARM_DIR') - - -@cached -def action_get(key=None): - """Gets the value of an action parameter, or all key/value param pairs""" - cmd = ['action-get'] - if key is not None: - cmd.append(key) - cmd.append('--format=json') - action_data = json.loads(subprocess.check_output(cmd).decode('UTF-8')) - return action_data - - -def action_set(values): - """Sets the values to be returned after the action finishes""" - cmd = ['action-set'] - for k, v in list(values.items()): - cmd.append('{}={}'.format(k, v)) - subprocess.check_call(cmd) - - -def action_fail(message): - """Sets the action status to failed and sets the error message. - - The results set by action_set are preserved.""" - subprocess.check_call(['action-fail', message]) - - -def action_name(): - """Get the name of the currently executing action.""" - return os.environ.get('JUJU_ACTION_NAME') - - -def action_uuid(): - """Get the UUID of the currently executing action.""" - return os.environ.get('JUJU_ACTION_UUID') - - -def action_tag(): - """Get the tag for the currently executing action.""" - return os.environ.get('JUJU_ACTION_TAG') - - -def status_set(workload_state, message): - """Set the workload state with a message - - Use status-set to set the workload state with a message which is visible - to the user via juju status. If the status-set command is not found then - assume this is juju < 1.23 and juju-log the message unstead. - - workload_state -- valid juju workload state. - message -- status update message - """ - valid_states = ['maintenance', 'blocked', 'waiting', 'active'] - if workload_state not in valid_states: - raise ValueError( - '{!r} is not a valid workload state'.format(workload_state) - ) - cmd = ['status-set', workload_state, message] - try: - ret = subprocess.call(cmd) - if ret == 0: - return - except OSError as e: - if e.errno != errno.ENOENT: - raise - log_message = 'status-set failed: {} {}'.format(workload_state, - message) - log(log_message, level='INFO') - - -def status_get(): - """Retrieve the previously set juju workload state and message - - If the status-get command is not found then assume this is juju < 1.23 and - return 'unknown', "" - - """ - cmd = ['status-get', "--format=json", "--include-data"] - try: - raw_status = subprocess.check_output(cmd) - except OSError as e: - if e.errno == errno.ENOENT: - return ('unknown', "") - else: - raise - else: - status = json.loads(raw_status.decode("UTF-8")) - return (status["status"], status["message"]) - - -def translate_exc(from_exc, to_exc): - def inner_translate_exc1(f): - @wraps(f) - def inner_translate_exc2(*args, **kwargs): - try: - return f(*args, **kwargs) - except from_exc: - raise to_exc - - return inner_translate_exc2 - - return inner_translate_exc1 - - -def application_version_set(version): - """Charm authors may trigger this command from any hook to output what - version of the application is running. This could be a package version, - for instance postgres version 9.5. It could also be a build number or - version control revision identifier, for instance git sha 6fb7ba68. """ - - cmd = ['application-version-set'] - cmd.append(version) - try: - subprocess.check_call(cmd) - except OSError: - log("Application Version: {}".format(version)) - - -@translate_exc(from_exc=OSError, to_exc=NotImplementedError) -def is_leader(): - """Does the current unit hold the juju leadership - - Uses juju to determine whether the current unit is the leader of its peers - """ - cmd = ['is-leader', '--format=json'] - return json.loads(subprocess.check_output(cmd).decode('UTF-8')) - - -@translate_exc(from_exc=OSError, to_exc=NotImplementedError) -def leader_get(attribute=None): - """Juju leader get value(s)""" - cmd = ['leader-get', '--format=json'] + [attribute or '-'] - return json.loads(subprocess.check_output(cmd).decode('UTF-8')) - - -@translate_exc(from_exc=OSError, to_exc=NotImplementedError) -def leader_set(settings=None, **kwargs): - """Juju leader set value(s)""" - # Don't log secrets. - # log("Juju leader-set '%s'" % (settings), level=DEBUG) - cmd = ['leader-set'] - settings = settings or {} - settings.update(kwargs) - for k, v in settings.items(): - if v is None: - cmd.append('{}='.format(k)) - else: - cmd.append('{}={}'.format(k, v)) - subprocess.check_call(cmd) - - -@translate_exc(from_exc=OSError, to_exc=NotImplementedError) -def payload_register(ptype, klass, pid): - """ is used while a hook is running to let Juju know that a - payload has been started.""" - cmd = ['payload-register'] - for x in [ptype, klass, pid]: - cmd.append(x) - subprocess.check_call(cmd) - - -@translate_exc(from_exc=OSError, to_exc=NotImplementedError) -def payload_unregister(klass, pid): - """ is used while a hook is running to let Juju know - that a payload has been manually stopped. The and provided - must match a payload that has been previously registered with juju using - payload-register.""" - cmd = ['payload-unregister'] - for x in [klass, pid]: - cmd.append(x) - subprocess.check_call(cmd) - - -@translate_exc(from_exc=OSError, to_exc=NotImplementedError) -def payload_status_set(klass, pid, status): - """is used to update the current status of a registered payload. - The and provided must match a payload that has been previously - registered with juju using payload-register. The must be one of the - follow: starting, started, stopping, stopped""" - cmd = ['payload-status-set'] - for x in [klass, pid, status]: - cmd.append(x) - subprocess.check_call(cmd) - - -@translate_exc(from_exc=OSError, to_exc=NotImplementedError) -def resource_get(name): - """used to fetch the resource path of the given name. - - must match a name of defined resource in metadata.yaml - - returns either a path or False if resource not available - """ - if not name: - return False - - cmd = ['resource-get', name] - try: - return subprocess.check_output(cmd).decode('UTF-8') - except subprocess.CalledProcessError: - return False - - -@cached -def juju_version(): - """Full version string (eg. '1.23.3.1-trusty-amd64')""" - # Per https://bugs.launchpad.net/juju-core/+bug/1455368/comments/1 - jujud = glob.glob('/var/lib/juju/tools/machine-*/jujud')[0] - return subprocess.check_output([jujud, 'version'], - universal_newlines=True).strip() - - -@cached -def has_juju_version(minimum_version): - """Return True if the Juju version is at least the provided version""" - return LooseVersion(juju_version()) >= LooseVersion(minimum_version) - - -_atexit = [] -_atstart = [] - - -def atstart(callback, *args, **kwargs): - '''Schedule a callback to run before the main hook. - - Callbacks are run in the order they were added. - - This is useful for modules and classes to perform initialization - and inject behavior. In particular: - - - Run common code before all of your hooks, such as logging - the hook name or interesting relation data. - - Defer object or module initialization that requires a hook - context until we know there actually is a hook context, - making testing easier. - - Rather than requiring charm authors to include boilerplate to - invoke your helper's behavior, have it run automatically if - your object is instantiated or module imported. - - This is not at all useful after your hook framework as been launched. - ''' - global _atstart - _atstart.append((callback, args, kwargs)) - - -def atexit(callback, *args, **kwargs): - '''Schedule a callback to run on successful hook completion. - - Callbacks are run in the reverse order that they were added.''' - _atexit.append((callback, args, kwargs)) - - -def _run_atstart(): - '''Hook frameworks must invoke this before running the main hook body.''' - global _atstart - for callback, args, kwargs in _atstart: - callback(*args, **kwargs) - del _atstart[:] - - -def _run_atexit(): - '''Hook frameworks must invoke this after the main hook body has - successfully completed. Do not invoke it if the hook fails.''' - global _atexit - for callback, args, kwargs in reversed(_atexit): - callback(*args, **kwargs) - del _atexit[:] - - -@translate_exc(from_exc=OSError, to_exc=NotImplementedError) -def network_get_primary_address(binding): - ''' - Retrieve the primary network address for a named binding - - :param binding: string. The name of a relation of extra-binding - :return: string. The primary IP address for the named binding - :raise: NotImplementedError if run on Juju < 2.0 - ''' - cmd = ['network-get', '--primary-address', binding] - return subprocess.check_output(cmd).decode('UTF-8').strip() - - -@translate_exc(from_exc=OSError, to_exc=NotImplementedError) -def network_get(endpoint, relation_id=None): - """ - Retrieve the network details for a relation endpoint - - :param endpoint: string. The name of a relation endpoint - :param relation_id: int. The ID of the relation for the current context. - :return: dict. The loaded YAML output of the network-get query. - :raise: NotImplementedError if run on Juju < 2.1 - """ - cmd = ['network-get', endpoint, '--format', 'yaml'] - if relation_id: - cmd.append('-r') - cmd.append(relation_id) - try: - response = subprocess.check_output( - cmd, - stderr=subprocess.STDOUT).decode('UTF-8').strip() - except CalledProcessError as e: - # Early versions of Juju 2.0.x required the --primary-address argument. - # We catch that condition here and raise NotImplementedError since - # the requested semantics are not available - the caller can then - # use the network_get_primary_address() method instead. - if '--primary-address is currently required' in e.output.decode('UTF-8'): - raise NotImplementedError - raise - return yaml.safe_load(response) - - -def add_metric(*args, **kwargs): - """Add metric values. Values may be expressed with keyword arguments. For - metric names containing dashes, these may be expressed as one or more - 'key=value' positional arguments. May only be called from the collect-metrics - hook.""" - _args = ['add-metric'] - _kvpairs = [] - _kvpairs.extend(args) - _kvpairs.extend(['{}={}'.format(k, v) for k, v in kwargs.items()]) - _args.extend(sorted(_kvpairs)) - try: - subprocess.check_call(_args) - return - except EnvironmentError as e: - if e.errno != errno.ENOENT: - raise - log_message = 'add-metric failed: {}'.format(' '.join(_kvpairs)) - log(log_message, level='INFO') - - -def meter_status(): - """Get the meter status, if running in the meter-status-changed hook.""" - return os.environ.get('JUJU_METER_STATUS') - - -def meter_info(): - """Get the meter status information, if running in the meter-status-changed - hook.""" - return os.environ.get('JUJU_METER_INFO') - - -def iter_units_for_relation_name(relation_name): - """Iterate through all units in a relation - - Generator that iterates through all the units in a relation and yields - a named tuple with rid and unit field names. - - Usage: - data = [(u.rid, u.unit) - for u in iter_units_for_relation_name(relation_name)] - - :param relation_name: string relation name - :yield: Named Tuple with rid and unit field names - """ - RelatedUnit = namedtuple('RelatedUnit', 'rid, unit') - for rid in relation_ids(relation_name): - for unit in related_units(rid): - yield RelatedUnit(rid, unit) - - -def ingress_address(rid=None, unit=None): - """ - Retrieve the ingress-address from a relation when available. Otherwise, - return the private-address. This function is to be used on the consuming - side of the relation. - - Usage: - addresses = [ingress_address(rid=u.rid, unit=u.unit) - for u in iter_units_for_relation_name(relation_name)] - - :param rid: string relation id - :param unit: string unit name - :side effect: calls relation_get - :return: string IP address - """ - settings = relation_get(rid=rid, unit=unit) - return (settings.get('ingress-address') or - settings.get('private-address')) diff --git a/tests/charmhelpers/core/host.py b/tests/charmhelpers/core/host.py deleted file mode 100644 index fd14d60..0000000 --- a/tests/charmhelpers/core/host.py +++ /dev/null @@ -1,1021 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Tools for working with the host system""" -# Copyright 2012 Canonical Ltd. -# -# Authors: -# Nick Moffitt -# Matthew Wedgwood - -import os -import re -import pwd -import glob -import grp -import random -import string -import subprocess -import hashlib -import functools -import itertools -import six - -from contextlib import contextmanager -from collections import OrderedDict -from .hookenv import log, DEBUG, local_unit -from .fstab import Fstab -from charmhelpers.osplatform import get_platform - -__platform__ = get_platform() -if __platform__ == "ubuntu": - from charmhelpers.core.host_factory.ubuntu import ( - service_available, - add_new_group, - lsb_release, - cmp_pkgrevno, - CompareHostReleases, - ) # flake8: noqa -- ignore F401 for this import -elif __platform__ == "centos": - from charmhelpers.core.host_factory.centos import ( - service_available, - add_new_group, - lsb_release, - cmp_pkgrevno, - CompareHostReleases, - ) # flake8: noqa -- ignore F401 for this import - -UPDATEDB_PATH = '/etc/updatedb.conf' - -def service_start(service_name, **kwargs): - """Start a system service. - - The specified service name is managed via the system level init system. - Some init systems (e.g. upstart) require that additional arguments be - provided in order to directly control service instances whereas other init - systems allow for addressing instances of a service directly by name (e.g. - systemd). - - The kwargs allow for the additional parameters to be passed to underlying - init systems for those systems which require/allow for them. For example, - the ceph-osd upstart script requires the id parameter to be passed along - in order to identify which running daemon should be reloaded. The follow- - ing example stops the ceph-osd service for instance id=4: - - service_stop('ceph-osd', id=4) - - :param service_name: the name of the service to stop - :param **kwargs: additional parameters to pass to the init system when - managing services. These will be passed as key=value - parameters to the init system's commandline. kwargs - are ignored for systemd enabled systems. - """ - return service('start', service_name, **kwargs) - - -def service_stop(service_name, **kwargs): - """Stop a system service. - - The specified service name is managed via the system level init system. - Some init systems (e.g. upstart) require that additional arguments be - provided in order to directly control service instances whereas other init - systems allow for addressing instances of a service directly by name (e.g. - systemd). - - The kwargs allow for the additional parameters to be passed to underlying - init systems for those systems which require/allow for them. For example, - the ceph-osd upstart script requires the id parameter to be passed along - in order to identify which running daemon should be reloaded. The follow- - ing example stops the ceph-osd service for instance id=4: - - service_stop('ceph-osd', id=4) - - :param service_name: the name of the service to stop - :param **kwargs: additional parameters to pass to the init system when - managing services. These will be passed as key=value - parameters to the init system's commandline. kwargs - are ignored for systemd enabled systems. - """ - return service('stop', service_name, **kwargs) - - -def service_restart(service_name, **kwargs): - """Restart a system service. - - The specified service name is managed via the system level init system. - Some init systems (e.g. upstart) require that additional arguments be - provided in order to directly control service instances whereas other init - systems allow for addressing instances of a service directly by name (e.g. - systemd). - - The kwargs allow for the additional parameters to be passed to underlying - init systems for those systems which require/allow for them. For example, - the ceph-osd upstart script requires the id parameter to be passed along - in order to identify which running daemon should be restarted. The follow- - ing example restarts the ceph-osd service for instance id=4: - - service_restart('ceph-osd', id=4) - - :param service_name: the name of the service to restart - :param **kwargs: additional parameters to pass to the init system when - managing services. These will be passed as key=value - parameters to the init system's commandline. kwargs - are ignored for init systems not allowing additional - parameters via the commandline (systemd). - """ - return service('restart', service_name) - - -def service_reload(service_name, restart_on_failure=False, **kwargs): - """Reload a system service, optionally falling back to restart if - reload fails. - - The specified service name is managed via the system level init system. - Some init systems (e.g. upstart) require that additional arguments be - provided in order to directly control service instances whereas other init - systems allow for addressing instances of a service directly by name (e.g. - systemd). - - The kwargs allow for the additional parameters to be passed to underlying - init systems for those systems which require/allow for them. For example, - the ceph-osd upstart script requires the id parameter to be passed along - in order to identify which running daemon should be reloaded. The follow- - ing example restarts the ceph-osd service for instance id=4: - - service_reload('ceph-osd', id=4) - - :param service_name: the name of the service to reload - :param restart_on_failure: boolean indicating whether to fallback to a - restart if the reload fails. - :param **kwargs: additional parameters to pass to the init system when - managing services. These will be passed as key=value - parameters to the init system's commandline. kwargs - are ignored for init systems not allowing additional - parameters via the commandline (systemd). - """ - service_result = service('reload', service_name, **kwargs) - if not service_result and restart_on_failure: - service_result = service('restart', service_name, **kwargs) - return service_result - - -def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d", - **kwargs): - """Pause a system service. - - Stop it, and prevent it from starting again at boot. - - :param service_name: the name of the service to pause - :param init_dir: path to the upstart init directory - :param initd_dir: path to the sysv init directory - :param **kwargs: additional parameters to pass to the init system when - managing services. These will be passed as key=value - parameters to the init system's commandline. kwargs - are ignored for init systems which do not support - key=value arguments via the commandline. - """ - stopped = True - if service_running(service_name, **kwargs): - stopped = service_stop(service_name, **kwargs) - upstart_file = os.path.join(init_dir, "{}.conf".format(service_name)) - sysv_file = os.path.join(initd_dir, service_name) - if init_is_systemd(): - service('disable', service_name) - service('mask', service_name) - elif os.path.exists(upstart_file): - override_path = os.path.join( - init_dir, '{}.override'.format(service_name)) - with open(override_path, 'w') as fh: - fh.write("manual\n") - elif os.path.exists(sysv_file): - subprocess.check_call(["update-rc.d", service_name, "disable"]) - else: - raise ValueError( - "Unable to detect {0} as SystemD, Upstart {1} or" - " SysV {2}".format( - service_name, upstart_file, sysv_file)) - return stopped - - -def service_resume(service_name, init_dir="/etc/init", - initd_dir="/etc/init.d", **kwargs): - """Resume a system service. - - Reenable starting again at boot. Start the service. - - :param service_name: the name of the service to resume - :param init_dir: the path to the init dir - :param initd dir: the path to the initd dir - :param **kwargs: additional parameters to pass to the init system when - managing services. These will be passed as key=value - parameters to the init system's commandline. kwargs - are ignored for systemd enabled systems. - """ - upstart_file = os.path.join(init_dir, "{}.conf".format(service_name)) - sysv_file = os.path.join(initd_dir, service_name) - if init_is_systemd(): - service('unmask', service_name) - service('enable', service_name) - elif os.path.exists(upstart_file): - override_path = os.path.join( - init_dir, '{}.override'.format(service_name)) - if os.path.exists(override_path): - os.unlink(override_path) - elif os.path.exists(sysv_file): - subprocess.check_call(["update-rc.d", service_name, "enable"]) - else: - raise ValueError( - "Unable to detect {0} as SystemD, Upstart {1} or" - " SysV {2}".format( - service_name, upstart_file, sysv_file)) - started = service_running(service_name, **kwargs) - - if not started: - started = service_start(service_name, **kwargs) - return started - - -def service(action, service_name, **kwargs): - """Control a system service. - - :param action: the action to take on the service - :param service_name: the name of the service to perform th action on - :param **kwargs: additional params to be passed to the service command in - the form of key=value. - """ - if init_is_systemd(): - cmd = ['systemctl', action, service_name] - else: - cmd = ['service', service_name, action] - for key, value in six.iteritems(kwargs): - parameter = '%s=%s' % (key, value) - cmd.append(parameter) - return subprocess.call(cmd) == 0 - - -_UPSTART_CONF = "/etc/init/{}.conf" -_INIT_D_CONF = "/etc/init.d/{}" - - -def service_running(service_name, **kwargs): - """Determine whether a system service is running. - - :param service_name: the name of the service - :param **kwargs: additional args to pass to the service command. This is - used to pass additional key=value arguments to the - service command line for managing specific instance - units (e.g. service ceph-osd status id=2). The kwargs - are ignored in systemd services. - """ - if init_is_systemd(): - return service('is-active', service_name) - else: - if os.path.exists(_UPSTART_CONF.format(service_name)): - try: - cmd = ['status', service_name] - for key, value in six.iteritems(kwargs): - parameter = '%s=%s' % (key, value) - cmd.append(parameter) - output = subprocess.check_output(cmd, - stderr=subprocess.STDOUT).decode('UTF-8') - except subprocess.CalledProcessError: - return False - else: - # This works for upstart scripts where the 'service' command - # returns a consistent string to represent running - # 'start/running' - if ("start/running" in output or - "is running" in output or - "up and running" in output): - return True - elif os.path.exists(_INIT_D_CONF.format(service_name)): - # Check System V scripts init script return codes - return service('status', service_name) - return False - - -SYSTEMD_SYSTEM = '/run/systemd/system' - - -def init_is_systemd(): - """Return True if the host system uses systemd, False otherwise.""" - if lsb_release()['DISTRIB_CODENAME'] == 'trusty': - return False - return os.path.isdir(SYSTEMD_SYSTEM) - - -def adduser(username, password=None, shell='/bin/bash', - system_user=False, primary_group=None, - secondary_groups=None, uid=None, home_dir=None): - """Add a user to the system. - - Will log but otherwise succeed if the user already exists. - - :param str username: Username to create - :param str password: Password for user; if ``None``, create a system user - :param str shell: The default shell for the user - :param bool system_user: Whether to create a login or system user - :param str primary_group: Primary group for user; defaults to username - :param list secondary_groups: Optional list of additional groups - :param int uid: UID for user being created - :param str home_dir: Home directory for user - - :returns: The password database entry struct, as returned by `pwd.getpwnam` - """ - try: - user_info = pwd.getpwnam(username) - log('user {0} already exists!'.format(username)) - if uid: - user_info = pwd.getpwuid(int(uid)) - log('user with uid {0} already exists!'.format(uid)) - except KeyError: - log('creating user {0}'.format(username)) - cmd = ['useradd'] - if uid: - cmd.extend(['--uid', str(uid)]) - if home_dir: - cmd.extend(['--home', str(home_dir)]) - if system_user or password is None: - cmd.append('--system') - else: - cmd.extend([ - '--create-home', - '--shell', shell, - '--password', password, - ]) - if not primary_group: - try: - grp.getgrnam(username) - primary_group = username # avoid "group exists" error - except KeyError: - pass - if primary_group: - cmd.extend(['-g', primary_group]) - if secondary_groups: - cmd.extend(['-G', ','.join(secondary_groups)]) - cmd.append(username) - subprocess.check_call(cmd) - user_info = pwd.getpwnam(username) - return user_info - - -def user_exists(username): - """Check if a user exists""" - try: - pwd.getpwnam(username) - user_exists = True - except KeyError: - user_exists = False - return user_exists - - -def uid_exists(uid): - """Check if a uid exists""" - try: - pwd.getpwuid(uid) - uid_exists = True - except KeyError: - uid_exists = False - return uid_exists - - -def group_exists(groupname): - """Check if a group exists""" - try: - grp.getgrnam(groupname) - group_exists = True - except KeyError: - group_exists = False - return group_exists - - -def gid_exists(gid): - """Check if a gid exists""" - try: - grp.getgrgid(gid) - gid_exists = True - except KeyError: - gid_exists = False - return gid_exists - - -def add_group(group_name, system_group=False, gid=None): - """Add a group to the system - - Will log but otherwise succeed if the group already exists. - - :param str group_name: group to create - :param bool system_group: Create system group - :param int gid: GID for user being created - - :returns: The password database entry struct, as returned by `grp.getgrnam` - """ - try: - group_info = grp.getgrnam(group_name) - log('group {0} already exists!'.format(group_name)) - if gid: - group_info = grp.getgrgid(gid) - log('group with gid {0} already exists!'.format(gid)) - except KeyError: - log('creating group {0}'.format(group_name)) - add_new_group(group_name, system_group, gid) - group_info = grp.getgrnam(group_name) - return group_info - - -def add_user_to_group(username, group): - """Add a user to a group""" - cmd = ['gpasswd', '-a', username, group] - log("Adding user {} to group {}".format(username, group)) - subprocess.check_call(cmd) - - -def chage(username, lastday=None, expiredate=None, inactive=None, - mindays=None, maxdays=None, root=None, warndays=None): - """Change user password expiry information - - :param str username: User to update - :param str lastday: Set when password was changed in YYYY-MM-DD format - :param str expiredate: Set when user's account will no longer be - accessible in YYYY-MM-DD format. - -1 will remove an account expiration date. - :param str inactive: Set the number of days of inactivity after a password - has expired before the account is locked. - -1 will remove an account's inactivity. - :param str mindays: Set the minimum number of days between password - changes to MIN_DAYS. - 0 indicates the password can be changed anytime. - :param str maxdays: Set the maximum number of days during which a - password is valid. - -1 as MAX_DAYS will remove checking maxdays - :param str root: Apply changes in the CHROOT_DIR directory - :param str warndays: Set the number of days of warning before a password - change is required - :raises subprocess.CalledProcessError: if call to chage fails - """ - cmd = ['chage'] - if root: - cmd.extend(['--root', root]) - if lastday: - cmd.extend(['--lastday', lastday]) - if expiredate: - cmd.extend(['--expiredate', expiredate]) - if inactive: - cmd.extend(['--inactive', inactive]) - if mindays: - cmd.extend(['--mindays', mindays]) - if maxdays: - cmd.extend(['--maxdays', maxdays]) - if warndays: - cmd.extend(['--warndays', warndays]) - cmd.append(username) - subprocess.check_call(cmd) - -remove_password_expiry = functools.partial(chage, expiredate='-1', inactive='-1', mindays='0', maxdays='-1') - -def rsync(from_path, to_path, flags='-r', options=None, timeout=None): - """Replicate the contents of a path""" - options = options or ['--delete', '--executability'] - cmd = ['/usr/bin/rsync', flags] - if timeout: - cmd = ['timeout', str(timeout)] + cmd - cmd.extend(options) - cmd.append(from_path) - cmd.append(to_path) - log(" ".join(cmd)) - return subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode('UTF-8').strip() - - -def symlink(source, destination): - """Create a symbolic link""" - log("Symlinking {} as {}".format(source, destination)) - cmd = [ - 'ln', - '-sf', - source, - destination, - ] - subprocess.check_call(cmd) - - -def mkdir(path, owner='root', group='root', perms=0o555, force=False): - """Create a directory""" - log("Making dir {} {}:{} {:o}".format(path, owner, group, - perms)) - uid = pwd.getpwnam(owner).pw_uid - gid = grp.getgrnam(group).gr_gid - realpath = os.path.abspath(path) - path_exists = os.path.exists(realpath) - if path_exists and force: - if not os.path.isdir(realpath): - log("Removing non-directory file {} prior to mkdir()".format(path)) - os.unlink(realpath) - os.makedirs(realpath, perms) - elif not path_exists: - os.makedirs(realpath, perms) - os.chown(realpath, uid, gid) - os.chmod(realpath, perms) - - -def write_file(path, content, owner='root', group='root', perms=0o444): - """Create or overwrite a file with the contents of a byte string.""" - uid = pwd.getpwnam(owner).pw_uid - gid = grp.getgrnam(group).gr_gid - # lets see if we can grab the file and compare the context, to avoid doing - # a write. - existing_content = None - existing_uid, existing_gid = None, None - try: - with open(path, 'rb') as target: - existing_content = target.read() - stat = os.stat(path) - existing_uid, existing_gid = stat.st_uid, stat.st_gid - except: - pass - if content != existing_content: - log("Writing file {} {}:{} {:o}".format(path, owner, group, perms), - level=DEBUG) - with open(path, 'wb') as target: - os.fchown(target.fileno(), uid, gid) - os.fchmod(target.fileno(), perms) - if six.PY3 and isinstance(content, six.string_types): - content = content.encode('UTF-8') - target.write(content) - return - # the contents were the same, but we might still need to change the - # ownership. - if existing_uid != uid: - log("Changing uid on already existing content: {} -> {}" - .format(existing_uid, uid), level=DEBUG) - os.chown(path, uid, -1) - if existing_gid != gid: - log("Changing gid on already existing content: {} -> {}" - .format(existing_gid, gid), level=DEBUG) - os.chown(path, -1, gid) - - -def fstab_remove(mp): - """Remove the given mountpoint entry from /etc/fstab""" - return Fstab.remove_by_mountpoint(mp) - - -def fstab_add(dev, mp, fs, options=None): - """Adds the given device entry to the /etc/fstab file""" - return Fstab.add(dev, mp, fs, options=options) - - -def mount(device, mountpoint, options=None, persist=False, filesystem="ext3"): - """Mount a filesystem at a particular mountpoint""" - cmd_args = ['mount'] - if options is not None: - cmd_args.extend(['-o', options]) - cmd_args.extend([device, mountpoint]) - try: - subprocess.check_output(cmd_args) - except subprocess.CalledProcessError as e: - log('Error mounting {} at {}\n{}'.format(device, mountpoint, e.output)) - return False - - if persist: - return fstab_add(device, mountpoint, filesystem, options=options) - return True - - -def umount(mountpoint, persist=False): - """Unmount a filesystem""" - cmd_args = ['umount', mountpoint] - try: - subprocess.check_output(cmd_args) - except subprocess.CalledProcessError as e: - log('Error unmounting {}\n{}'.format(mountpoint, e.output)) - return False - - if persist: - return fstab_remove(mountpoint) - return True - - -def mounts(): - """Get a list of all mounted volumes as [[mountpoint,device],[...]]""" - with open('/proc/mounts') as f: - # [['/mount/point','/dev/path'],[...]] - system_mounts = [m[1::-1] for m in [l.strip().split() - for l in f.readlines()]] - return system_mounts - - -def fstab_mount(mountpoint): - """Mount filesystem using fstab""" - cmd_args = ['mount', mountpoint] - try: - subprocess.check_output(cmd_args) - except subprocess.CalledProcessError as e: - log('Error unmounting {}\n{}'.format(mountpoint, e.output)) - return False - return True - - -def file_hash(path, hash_type='md5'): - """Generate a hash checksum of the contents of 'path' or None if not found. - - :param str hash_type: Any hash alrgorithm supported by :mod:`hashlib`, - such as md5, sha1, sha256, sha512, etc. - """ - if os.path.exists(path): - h = getattr(hashlib, hash_type)() - with open(path, 'rb') as source: - h.update(source.read()) - return h.hexdigest() - else: - return None - - -def path_hash(path): - """Generate a hash checksum of all files matching 'path'. Standard - wildcards like '*' and '?' are supported, see documentation for the 'glob' - module for more information. - - :return: dict: A { filename: hash } dictionary for all matched files. - Empty if none found. - """ - return { - filename: file_hash(filename) - for filename in glob.iglob(path) - } - - -def check_hash(path, checksum, hash_type='md5'): - """Validate a file using a cryptographic checksum. - - :param str checksum: Value of the checksum used to validate the file. - :param str hash_type: Hash algorithm used to generate `checksum`. - Can be any hash alrgorithm supported by :mod:`hashlib`, - such as md5, sha1, sha256, sha512, etc. - :raises ChecksumError: If the file fails the checksum - - """ - actual_checksum = file_hash(path, hash_type) - if checksum != actual_checksum: - raise ChecksumError("'%s' != '%s'" % (checksum, actual_checksum)) - - -class ChecksumError(ValueError): - """A class derived from Value error to indicate the checksum failed.""" - pass - - -def restart_on_change(restart_map, stopstart=False, restart_functions=None): - """Restart services based on configuration files changing - - This function is used a decorator, for example:: - - @restart_on_change({ - '/etc/ceph/ceph.conf': [ 'cinder-api', 'cinder-volume' ] - '/etc/apache/sites-enabled/*': [ 'apache2' ] - }) - def config_changed(): - pass # your code here - - In this example, the cinder-api and cinder-volume services - would be restarted if /etc/ceph/ceph.conf is changed by the - ceph_client_changed function. The apache2 service would be - restarted if any file matching the pattern got changed, created - or removed. Standard wildcards are supported, see documentation - for the 'glob' module for more information. - - @param restart_map: {path_file_name: [service_name, ...] - @param stopstart: DEFAULT false; whether to stop, start OR restart - @param restart_functions: nonstandard functions to use to restart services - {svc: func, ...} - @returns result from decorated function - """ - def wrap(f): - @functools.wraps(f) - def wrapped_f(*args, **kwargs): - return restart_on_change_helper( - (lambda: f(*args, **kwargs)), restart_map, stopstart, - restart_functions) - return wrapped_f - return wrap - - -def restart_on_change_helper(lambda_f, restart_map, stopstart=False, - restart_functions=None): - """Helper function to perform the restart_on_change function. - - This is provided for decorators to restart services if files described - in the restart_map have changed after an invocation of lambda_f(). - - @param lambda_f: function to call. - @param restart_map: {file: [service, ...]} - @param stopstart: whether to stop, start or restart a service - @param restart_functions: nonstandard functions to use to restart services - {svc: func, ...} - @returns result of lambda_f() - """ - if restart_functions is None: - restart_functions = {} - checksums = {path: path_hash(path) for path in restart_map} - r = lambda_f() - # create a list of lists of the services to restart - restarts = [restart_map[path] - for path in restart_map - if path_hash(path) != checksums[path]] - # create a flat list of ordered services without duplicates from lists - services_list = list(OrderedDict.fromkeys(itertools.chain(*restarts))) - if services_list: - actions = ('stop', 'start') if stopstart else ('restart',) - for service_name in services_list: - if service_name in restart_functions: - restart_functions[service_name](service_name) - else: - for action in actions: - service(action, service_name) - return r - - -def pwgen(length=None): - """Generate a random pasword.""" - if length is None: - # A random length is ok to use a weak PRNG - length = random.choice(range(35, 45)) - alphanumeric_chars = [ - l for l in (string.ascii_letters + string.digits) - if l not in 'l0QD1vAEIOUaeiou'] - # Use a crypto-friendly PRNG (e.g. /dev/urandom) for making the - # actual password - random_generator = random.SystemRandom() - random_chars = [ - random_generator.choice(alphanumeric_chars) for _ in range(length)] - return(''.join(random_chars)) - - -def is_phy_iface(interface): - """Returns True if interface is not virtual, otherwise False.""" - if interface: - sys_net = '/sys/class/net' - if os.path.isdir(sys_net): - for iface in glob.glob(os.path.join(sys_net, '*')): - if '/virtual/' in os.path.realpath(iface): - continue - - if interface == os.path.basename(iface): - return True - - return False - - -def get_bond_master(interface): - """Returns bond master if interface is bond slave otherwise None. - - NOTE: the provided interface is expected to be physical - """ - if interface: - iface_path = '/sys/class/net/%s' % (interface) - if os.path.exists(iface_path): - if '/virtual/' in os.path.realpath(iface_path): - return None - - master = os.path.join(iface_path, 'master') - if os.path.exists(master): - master = os.path.realpath(master) - # make sure it is a bond master - if os.path.exists(os.path.join(master, 'bonding')): - return os.path.basename(master) - - return None - - -def list_nics(nic_type=None): - """Return a list of nics of given type(s)""" - if isinstance(nic_type, six.string_types): - int_types = [nic_type] - else: - int_types = nic_type - - interfaces = [] - if nic_type: - for int_type in int_types: - cmd = ['ip', 'addr', 'show', 'label', int_type + '*'] - ip_output = subprocess.check_output(cmd).decode('UTF-8') - ip_output = ip_output.split('\n') - ip_output = (line for line in ip_output if line) - for line in ip_output: - if line.split()[1].startswith(int_type): - matched = re.search('.*: (' + int_type + - r'[0-9]+\.[0-9]+)@.*', line) - if matched: - iface = matched.groups()[0] - else: - iface = line.split()[1].replace(":", "") - - if iface not in interfaces: - interfaces.append(iface) - else: - cmd = ['ip', 'a'] - ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n') - ip_output = (line.strip() for line in ip_output if line) - - key = re.compile('^[0-9]+:\s+(.+):') - for line in ip_output: - matched = re.search(key, line) - if matched: - iface = matched.group(1) - iface = iface.partition("@")[0] - if iface not in interfaces: - interfaces.append(iface) - - return interfaces - - -def set_nic_mtu(nic, mtu): - """Set the Maximum Transmission Unit (MTU) on a network interface.""" - cmd = ['ip', 'link', 'set', nic, 'mtu', mtu] - subprocess.check_call(cmd) - - -def get_nic_mtu(nic): - """Return the Maximum Transmission Unit (MTU) for a network interface.""" - cmd = ['ip', 'addr', 'show', nic] - ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n') - mtu = "" - for line in ip_output: - words = line.split() - if 'mtu' in words: - mtu = words[words.index("mtu") + 1] - return mtu - - -def get_nic_hwaddr(nic): - """Return the Media Access Control (MAC) for a network interface.""" - cmd = ['ip', '-o', '-0', 'addr', 'show', nic] - ip_output = subprocess.check_output(cmd).decode('UTF-8') - hwaddr = "" - words = ip_output.split() - if 'link/ether' in words: - hwaddr = words[words.index('link/ether') + 1] - return hwaddr - - -@contextmanager -def chdir(directory): - """Change the current working directory to a different directory for a code - block and return the previous directory after the block exits. Useful to - run commands from a specificed directory. - - :param str directory: The directory path to change to for this context. - """ - cur = os.getcwd() - try: - yield os.chdir(directory) - finally: - os.chdir(cur) - - -def chownr(path, owner, group, follow_links=True, chowntopdir=False): - """Recursively change user and group ownership of files and directories - in given path. Doesn't chown path itself by default, only its children. - - :param str path: The string path to start changing ownership. - :param str owner: The owner string to use when looking up the uid. - :param str group: The group string to use when looking up the gid. - :param bool follow_links: Also follow and chown links if True - :param bool chowntopdir: Also chown path itself if True - """ - uid = pwd.getpwnam(owner).pw_uid - gid = grp.getgrnam(group).gr_gid - if follow_links: - chown = os.chown - else: - chown = os.lchown - - if chowntopdir: - broken_symlink = os.path.lexists(path) and not os.path.exists(path) - if not broken_symlink: - chown(path, uid, gid) - for root, dirs, files in os.walk(path, followlinks=follow_links): - for name in dirs + files: - full = os.path.join(root, name) - broken_symlink = os.path.lexists(full) and not os.path.exists(full) - if not broken_symlink: - chown(full, uid, gid) - - -def lchownr(path, owner, group): - """Recursively change user and group ownership of files and directories - in a given path, not following symbolic links. See the documentation for - 'os.lchown' for more information. - - :param str path: The string path to start changing ownership. - :param str owner: The owner string to use when looking up the uid. - :param str group: The group string to use when looking up the gid. - """ - chownr(path, owner, group, follow_links=False) - - -def owner(path): - """Returns a tuple containing the username & groupname owning the path. - - :param str path: the string path to retrieve the ownership - :return tuple(str, str): A (username, groupname) tuple containing the - name of the user and group owning the path. - :raises OSError: if the specified path does not exist - """ - stat = os.stat(path) - username = pwd.getpwuid(stat.st_uid)[0] - groupname = grp.getgrgid(stat.st_gid)[0] - return username, groupname - - -def get_total_ram(): - """The total amount of system RAM in bytes. - - This is what is reported by the OS, and may be overcommitted when - there are multiple containers hosted on the same machine. - """ - with open('/proc/meminfo', 'r') as f: - for line in f.readlines(): - if line: - key, value, unit = line.split() - if key == 'MemTotal:': - assert unit == 'kB', 'Unknown unit' - return int(value) * 1024 # Classic, not KiB. - raise NotImplementedError() - - -UPSTART_CONTAINER_TYPE = '/run/container_type' - - -def is_container(): - """Determine whether unit is running in a container - - @return: boolean indicating if unit is in a container - """ - if init_is_systemd(): - # Detect using systemd-detect-virt - return subprocess.call(['systemd-detect-virt', - '--container']) == 0 - else: - # Detect using upstart container file marker - return os.path.exists(UPSTART_CONTAINER_TYPE) - - -def add_to_updatedb_prunepath(path, updatedb_path=UPDATEDB_PATH): - with open(updatedb_path, 'r+') as f_id: - updatedb_text = f_id.read() - output = updatedb(updatedb_text, path) - f_id.seek(0) - f_id.write(output) - f_id.truncate() - - -def updatedb(updatedb_text, new_path): - lines = [line for line in updatedb_text.split("\n")] - for i, line in enumerate(lines): - if line.startswith("PRUNEPATHS="): - paths_line = line.split("=")[1].replace('"', '') - paths = paths_line.split(" ") - if new_path not in paths: - paths.append(new_path) - lines[i] = 'PRUNEPATHS="{}"'.format(' '.join(paths)) - output = "\n".join(lines) - return output - - -def modulo_distribution(modulo=3, wait=30): - """ Modulo distribution - - This helper uses the unit number, a modulo value and a constant wait time - to produce a calculated wait time distribution. This is useful in large - scale deployments to distribute load during an expensive operation such as - service restarts. - - If you have 1000 nodes that need to restart 100 at a time 1 minute at a - time: - - time.wait(modulo_distribution(modulo=100, wait=60)) - restart() - - If you need restarts to happen serially set modulo to the exact number of - nodes and set a high constant wait time: - - time.wait(modulo_distribution(modulo=10, wait=120)) - restart() - - @param modulo: int The modulo number creates the group distribution - @param wait: int The constant time wait value - @return: int Calculated time to wait for unit operation - """ - unit_number = int(local_unit().split('/')[1]) - return (unit_number % modulo) * wait diff --git a/tests/charmhelpers/core/host_factory/__init__.py b/tests/charmhelpers/core/host_factory/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/charmhelpers/core/host_factory/centos.py b/tests/charmhelpers/core/host_factory/centos.py deleted file mode 100644 index 7781a39..0000000 --- a/tests/charmhelpers/core/host_factory/centos.py +++ /dev/null @@ -1,72 +0,0 @@ -import subprocess -import yum -import os - -from charmhelpers.core.strutils import BasicStringComparator - - -class CompareHostReleases(BasicStringComparator): - """Provide comparisons of Host releases. - - Use in the form of - - if CompareHostReleases(release) > 'trusty': - # do something with mitaka - """ - - def __init__(self, item): - raise NotImplementedError( - "CompareHostReleases() is not implemented for CentOS") - - -def service_available(service_name): - # """Determine whether a system service is available.""" - if os.path.isdir('/run/systemd/system'): - cmd = ['systemctl', 'is-enabled', service_name] - else: - cmd = ['service', service_name, 'is-enabled'] - return subprocess.call(cmd) == 0 - - -def add_new_group(group_name, system_group=False, gid=None): - cmd = ['groupadd'] - if gid: - cmd.extend(['--gid', str(gid)]) - if system_group: - cmd.append('-r') - cmd.append(group_name) - subprocess.check_call(cmd) - - -def lsb_release(): - """Return /etc/os-release in a dict.""" - d = {} - with open('/etc/os-release', 'r') as lsb: - for l in lsb: - s = l.split('=') - if len(s) != 2: - continue - d[s[0].strip()] = s[1].strip() - return d - - -def cmp_pkgrevno(package, revno, pkgcache=None): - """Compare supplied revno with the revno of the installed package. - - * 1 => Installed revno is greater than supplied arg - * 0 => Installed revno is the same as supplied arg - * -1 => Installed revno is less than supplied arg - - This function imports YumBase function if the pkgcache argument - is None. - """ - if not pkgcache: - y = yum.YumBase() - packages = y.doPackageLists() - pkgcache = {i.Name: i.version for i in packages['installed']} - pkg = pkgcache[package] - if pkg > revno: - return 1 - if pkg < revno: - return -1 - return 0 diff --git a/tests/charmhelpers/core/host_factory/ubuntu.py b/tests/charmhelpers/core/host_factory/ubuntu.py deleted file mode 100644 index 99451b5..0000000 --- a/tests/charmhelpers/core/host_factory/ubuntu.py +++ /dev/null @@ -1,90 +0,0 @@ -import subprocess - -from charmhelpers.core.strutils import BasicStringComparator - - -UBUNTU_RELEASES = ( - 'lucid', - 'maverick', - 'natty', - 'oneiric', - 'precise', - 'quantal', - 'raring', - 'saucy', - 'trusty', - 'utopic', - 'vivid', - 'wily', - 'xenial', - 'yakkety', - 'zesty', - 'artful', - 'bionic', -) - - -class CompareHostReleases(BasicStringComparator): - """Provide comparisons of Ubuntu releases. - - Use in the form of - - if CompareHostReleases(release) > 'trusty': - # do something with mitaka - """ - _list = UBUNTU_RELEASES - - -def service_available(service_name): - """Determine whether a system service is available""" - try: - subprocess.check_output( - ['service', service_name, 'status'], - stderr=subprocess.STDOUT).decode('UTF-8') - except subprocess.CalledProcessError as e: - return b'unrecognized service' not in e.output - else: - return True - - -def add_new_group(group_name, system_group=False, gid=None): - cmd = ['addgroup'] - if gid: - cmd.extend(['--gid', str(gid)]) - if system_group: - cmd.append('--system') - else: - cmd.extend([ - '--group', - ]) - cmd.append(group_name) - subprocess.check_call(cmd) - - -def lsb_release(): - """Return /etc/lsb-release in a dict""" - d = {} - with open('/etc/lsb-release', 'r') as lsb: - for l in lsb: - k, v = l.split('=') - d[k.strip()] = v.strip() - return d - - -def cmp_pkgrevno(package, revno, pkgcache=None): - """Compare supplied revno with the revno of the installed package. - - * 1 => Installed revno is greater than supplied arg - * 0 => Installed revno is the same as supplied arg - * -1 => Installed revno is less than supplied arg - - This function imports apt_cache function from charmhelpers.fetch if - the pkgcache argument is None. Be sure to add charmhelpers.fetch if - you call this function, or pass an apt_pkg.Cache() instance. - """ - import apt_pkg - if not pkgcache: - from charmhelpers.fetch import apt_cache - pkgcache = apt_cache() - pkg = pkgcache[package] - return apt_pkg.version_compare(pkg.current_ver.ver_str, revno) diff --git a/tests/charmhelpers/core/hugepage.py b/tests/charmhelpers/core/hugepage.py deleted file mode 100644 index 54b5b5e..0000000 --- a/tests/charmhelpers/core/hugepage.py +++ /dev/null @@ -1,69 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import yaml -from charmhelpers.core import fstab -from charmhelpers.core import sysctl -from charmhelpers.core.host import ( - add_group, - add_user_to_group, - fstab_mount, - mkdir, -) -from charmhelpers.core.strutils import bytes_from_string -from subprocess import check_output - - -def hugepage_support(user, group='hugetlb', nr_hugepages=256, - max_map_count=65536, mnt_point='/run/hugepages/kvm', - pagesize='2MB', mount=True, set_shmmax=False): - """Enable hugepages on system. - - Args: - user (str) -- Username to allow access to hugepages to - group (str) -- Group name to own hugepages - nr_hugepages (int) -- Number of pages to reserve - max_map_count (int) -- Number of Virtual Memory Areas a process can own - mnt_point (str) -- Directory to mount hugepages on - pagesize (str) -- Size of hugepages - mount (bool) -- Whether to Mount hugepages - """ - group_info = add_group(group) - gid = group_info.gr_gid - add_user_to_group(user, group) - if max_map_count < 2 * nr_hugepages: - max_map_count = 2 * nr_hugepages - sysctl_settings = { - 'vm.nr_hugepages': nr_hugepages, - 'vm.max_map_count': max_map_count, - 'vm.hugetlb_shm_group': gid, - } - if set_shmmax: - shmmax_current = int(check_output(['sysctl', '-n', 'kernel.shmmax'])) - shmmax_minsize = bytes_from_string(pagesize) * nr_hugepages - if shmmax_minsize > shmmax_current: - sysctl_settings['kernel.shmmax'] = shmmax_minsize - sysctl.create(yaml.dump(sysctl_settings), '/etc/sysctl.d/10-hugepage.conf') - mkdir(mnt_point, owner='root', group='root', perms=0o755, force=False) - lfstab = fstab.Fstab() - fstab_entry = lfstab.get_entry_by_attr('mountpoint', mnt_point) - if fstab_entry: - lfstab.remove_entry(fstab_entry) - entry = lfstab.Entry('nodev', mnt_point, 'hugetlbfs', - 'mode=1770,gid={},pagesize={}'.format(gid, pagesize), 0, 0) - lfstab.add_entry(entry) - if mount: - fstab_mount(mnt_point) diff --git a/tests/charmhelpers/core/kernel.py b/tests/charmhelpers/core/kernel.py deleted file mode 100644 index 2d40452..0000000 --- a/tests/charmhelpers/core/kernel.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -import subprocess - -from charmhelpers.osplatform import get_platform -from charmhelpers.core.hookenv import ( - log, - INFO -) - -__platform__ = get_platform() -if __platform__ == "ubuntu": - from charmhelpers.core.kernel_factory.ubuntu import ( - persistent_modprobe, - update_initramfs, - ) # flake8: noqa -- ignore F401 for this import -elif __platform__ == "centos": - from charmhelpers.core.kernel_factory.centos import ( - persistent_modprobe, - update_initramfs, - ) # flake8: noqa -- ignore F401 for this import - -__author__ = "Jorge Niedbalski " - - -def modprobe(module, persist=True): - """Load a kernel module and configure for auto-load on reboot.""" - cmd = ['modprobe', module] - - log('Loading kernel module %s' % module, level=INFO) - - subprocess.check_call(cmd) - if persist: - persistent_modprobe(module) - - -def rmmod(module, force=False): - """Remove a module from the linux kernel""" - cmd = ['rmmod'] - if force: - cmd.append('-f') - cmd.append(module) - log('Removing kernel module %s' % module, level=INFO) - return subprocess.check_call(cmd) - - -def lsmod(): - """Shows what kernel modules are currently loaded""" - return subprocess.check_output(['lsmod'], - universal_newlines=True) - - -def is_module_loaded(module): - """Checks if a kernel module is already loaded""" - matches = re.findall('^%s[ ]+' % module, lsmod(), re.M) - return len(matches) > 0 diff --git a/tests/charmhelpers/core/kernel_factory/__init__.py b/tests/charmhelpers/core/kernel_factory/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/charmhelpers/core/kernel_factory/centos.py b/tests/charmhelpers/core/kernel_factory/centos.py deleted file mode 100644 index 1c402c1..0000000 --- a/tests/charmhelpers/core/kernel_factory/centos.py +++ /dev/null @@ -1,17 +0,0 @@ -import subprocess -import os - - -def persistent_modprobe(module): - """Load a kernel module and configure for auto-load on reboot.""" - if not os.path.exists('/etc/rc.modules'): - open('/etc/rc.modules', 'a') - os.chmod('/etc/rc.modules', 111) - with open('/etc/rc.modules', 'r+') as modules: - if module not in modules.read(): - modules.write('modprobe %s\n' % module) - - -def update_initramfs(version='all'): - """Updates an initramfs image.""" - return subprocess.check_call(["dracut", "-f", version]) diff --git a/tests/charmhelpers/core/kernel_factory/ubuntu.py b/tests/charmhelpers/core/kernel_factory/ubuntu.py deleted file mode 100644 index 3de372f..0000000 --- a/tests/charmhelpers/core/kernel_factory/ubuntu.py +++ /dev/null @@ -1,13 +0,0 @@ -import subprocess - - -def persistent_modprobe(module): - """Load a kernel module and configure for auto-load on reboot.""" - with open('/etc/modules', 'r+') as modules: - if module not in modules.read(): - modules.write(module + "\n") - - -def update_initramfs(version='all'): - """Updates an initramfs image.""" - return subprocess.check_call(["update-initramfs", "-k", version, "-u"]) diff --git a/tests/charmhelpers/core/services/__init__.py b/tests/charmhelpers/core/services/__init__.py deleted file mode 100644 index 61fd074..0000000 --- a/tests/charmhelpers/core/services/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .base import * # NOQA -from .helpers import * # NOQA diff --git a/tests/charmhelpers/core/services/base.py b/tests/charmhelpers/core/services/base.py deleted file mode 100644 index ca9dc99..0000000 --- a/tests/charmhelpers/core/services/base.py +++ /dev/null @@ -1,351 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import json -from inspect import getargspec -from collections import Iterable, OrderedDict - -from charmhelpers.core import host -from charmhelpers.core import hookenv - - -__all__ = ['ServiceManager', 'ManagerCallback', - 'PortManagerCallback', 'open_ports', 'close_ports', 'manage_ports', - 'service_restart', 'service_stop'] - - -class ServiceManager(object): - def __init__(self, services=None): - """ - Register a list of services, given their definitions. - - Service definitions are dicts in the following formats (all keys except - 'service' are optional):: - - { - "service": , - "required_data": , - "provided_data": , - "data_ready": , - "data_lost": , - "start": , - "stop": , - "ports": , - } - - The 'required_data' list should contain dicts of required data (or - dependency managers that act like dicts and know how to collect the data). - Only when all items in the 'required_data' list are populated are the list - of 'data_ready' and 'start' callbacks executed. See `is_ready()` for more - information. - - The 'provided_data' list should contain relation data providers, most likely - a subclass of :class:`charmhelpers.core.services.helpers.RelationContext`, - that will indicate a set of data to set on a given relation. - - The 'data_ready' value should be either a single callback, or a list of - callbacks, to be called when all items in 'required_data' pass `is_ready()`. - Each callback will be called with the service name as the only parameter. - After all of the 'data_ready' callbacks are called, the 'start' callbacks - are fired. - - The 'data_lost' value should be either a single callback, or a list of - callbacks, to be called when a 'required_data' item no longer passes - `is_ready()`. Each callback will be called with the service name as the - only parameter. After all of the 'data_lost' callbacks are called, - the 'stop' callbacks are fired. - - The 'start' value should be either a single callback, or a list of - callbacks, to be called when starting the service, after the 'data_ready' - callbacks are complete. Each callback will be called with the service - name as the only parameter. This defaults to - `[host.service_start, services.open_ports]`. - - The 'stop' value should be either a single callback, or a list of - callbacks, to be called when stopping the service. If the service is - being stopped because it no longer has all of its 'required_data', this - will be called after all of the 'data_lost' callbacks are complete. - Each callback will be called with the service name as the only parameter. - This defaults to `[services.close_ports, host.service_stop]`. - - The 'ports' value should be a list of ports to manage. The default - 'start' handler will open the ports after the service is started, - and the default 'stop' handler will close the ports prior to stopping - the service. - - - Examples: - - The following registers an Upstart service called bingod that depends on - a mongodb relation and which runs a custom `db_migrate` function prior to - restarting the service, and a Runit service called spadesd:: - - manager = services.ServiceManager([ - { - 'service': 'bingod', - 'ports': [80, 443], - 'required_data': [MongoRelation(), config(), {'my': 'data'}], - 'data_ready': [ - services.template(source='bingod.conf'), - services.template(source='bingod.ini', - target='/etc/bingod.ini', - owner='bingo', perms=0400), - ], - }, - { - 'service': 'spadesd', - 'data_ready': services.template(source='spadesd_run.j2', - target='/etc/sv/spadesd/run', - perms=0555), - 'start': runit_start, - 'stop': runit_stop, - }, - ]) - manager.manage() - """ - self._ready_file = os.path.join(hookenv.charm_dir(), 'READY-SERVICES.json') - self._ready = None - self.services = OrderedDict() - for service in services or []: - service_name = service['service'] - self.services[service_name] = service - - def manage(self): - """ - Handle the current hook by doing The Right Thing with the registered services. - """ - hookenv._run_atstart() - try: - hook_name = hookenv.hook_name() - if hook_name == 'stop': - self.stop_services() - else: - self.reconfigure_services() - self.provide_data() - except SystemExit as x: - if x.code is None or x.code == 0: - hookenv._run_atexit() - hookenv._run_atexit() - - def provide_data(self): - """ - Set the relation data for each provider in the ``provided_data`` list. - - A provider must have a `name` attribute, which indicates which relation - to set data on, and a `provide_data()` method, which returns a dict of - data to set. - - The `provide_data()` method can optionally accept two parameters: - - * ``remote_service`` The name of the remote service that the data will - be provided to. The `provide_data()` method will be called once - for each connected service (not unit). This allows the method to - tailor its data to the given service. - * ``service_ready`` Whether or not the service definition had all of - its requirements met, and thus the ``data_ready`` callbacks run. - - Note that the ``provided_data`` methods are now called **after** the - ``data_ready`` callbacks are run. This gives the ``data_ready`` callbacks - a chance to generate any data necessary for the providing to the remote - services. - """ - for service_name, service in self.services.items(): - service_ready = self.is_ready(service_name) - for provider in service.get('provided_data', []): - for relid in hookenv.relation_ids(provider.name): - units = hookenv.related_units(relid) - if not units: - continue - remote_service = units[0].split('/')[0] - argspec = getargspec(provider.provide_data) - if len(argspec.args) > 1: - data = provider.provide_data(remote_service, service_ready) - else: - data = provider.provide_data() - if data: - hookenv.relation_set(relid, data) - - def reconfigure_services(self, *service_names): - """ - Update all files for one or more registered services, and, - if ready, optionally restart them. - - If no service names are given, reconfigures all registered services. - """ - for service_name in service_names or self.services.keys(): - if self.is_ready(service_name): - self.fire_event('data_ready', service_name) - self.fire_event('start', service_name, default=[ - service_restart, - manage_ports]) - self.save_ready(service_name) - else: - if self.was_ready(service_name): - self.fire_event('data_lost', service_name) - self.fire_event('stop', service_name, default=[ - manage_ports, - service_stop]) - self.save_lost(service_name) - - def stop_services(self, *service_names): - """ - Stop one or more registered services, by name. - - If no service names are given, stops all registered services. - """ - for service_name in service_names or self.services.keys(): - self.fire_event('stop', service_name, default=[ - manage_ports, - service_stop]) - - def get_service(self, service_name): - """ - Given the name of a registered service, return its service definition. - """ - service = self.services.get(service_name) - if not service: - raise KeyError('Service not registered: %s' % service_name) - return service - - def fire_event(self, event_name, service_name, default=None): - """ - Fire a data_ready, data_lost, start, or stop event on a given service. - """ - service = self.get_service(service_name) - callbacks = service.get(event_name, default) - if not callbacks: - return - if not isinstance(callbacks, Iterable): - callbacks = [callbacks] - for callback in callbacks: - if isinstance(callback, ManagerCallback): - callback(self, service_name, event_name) - else: - callback(service_name) - - def is_ready(self, service_name): - """ - Determine if a registered service is ready, by checking its 'required_data'. - - A 'required_data' item can be any mapping type, and is considered ready - if `bool(item)` evaluates as True. - """ - service = self.get_service(service_name) - reqs = service.get('required_data', []) - return all(bool(req) for req in reqs) - - def _load_ready_file(self): - if self._ready is not None: - return - if os.path.exists(self._ready_file): - with open(self._ready_file) as fp: - self._ready = set(json.load(fp)) - else: - self._ready = set() - - def _save_ready_file(self): - if self._ready is None: - return - with open(self._ready_file, 'w') as fp: - json.dump(list(self._ready), fp) - - def save_ready(self, service_name): - """ - Save an indicator that the given service is now data_ready. - """ - self._load_ready_file() - self._ready.add(service_name) - self._save_ready_file() - - def save_lost(self, service_name): - """ - Save an indicator that the given service is no longer data_ready. - """ - self._load_ready_file() - self._ready.discard(service_name) - self._save_ready_file() - - def was_ready(self, service_name): - """ - Determine if the given service was previously data_ready. - """ - self._load_ready_file() - return service_name in self._ready - - -class ManagerCallback(object): - """ - Special case of a callback that takes the `ServiceManager` instance - in addition to the service name. - - Subclasses should implement `__call__` which should accept three parameters: - - * `manager` The `ServiceManager` instance - * `service_name` The name of the service it's being triggered for - * `event_name` The name of the event that this callback is handling - """ - def __call__(self, manager, service_name, event_name): - raise NotImplementedError() - - -class PortManagerCallback(ManagerCallback): - """ - Callback class that will open or close ports, for use as either - a start or stop action. - """ - def __call__(self, manager, service_name, event_name): - service = manager.get_service(service_name) - new_ports = service.get('ports', []) - port_file = os.path.join(hookenv.charm_dir(), '.{}.ports'.format(service_name)) - if os.path.exists(port_file): - with open(port_file) as fp: - old_ports = fp.read().split(',') - for old_port in old_ports: - if bool(old_port): - old_port = int(old_port) - if old_port not in new_ports: - hookenv.close_port(old_port) - with open(port_file, 'w') as fp: - fp.write(','.join(str(port) for port in new_ports)) - for port in new_ports: - if event_name == 'start': - hookenv.open_port(port) - elif event_name == 'stop': - hookenv.close_port(port) - - -def service_stop(service_name): - """ - Wrapper around host.service_stop to prevent spurious "unknown service" - messages in the logs. - """ - if host.service_running(service_name): - host.service_stop(service_name) - - -def service_restart(service_name): - """ - Wrapper around host.service_restart to prevent spurious "unknown service" - messages in the logs. - """ - if host.service_available(service_name): - if host.service_running(service_name): - host.service_restart(service_name) - else: - host.service_start(service_name) - - -# Convenience aliases -open_ports = close_ports = manage_ports = PortManagerCallback() diff --git a/tests/charmhelpers/core/services/helpers.py b/tests/charmhelpers/core/services/helpers.py deleted file mode 100644 index 3e6e30d..0000000 --- a/tests/charmhelpers/core/services/helpers.py +++ /dev/null @@ -1,290 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import yaml - -from charmhelpers.core import hookenv -from charmhelpers.core import host -from charmhelpers.core import templating - -from charmhelpers.core.services.base import ManagerCallback - - -__all__ = ['RelationContext', 'TemplateCallback', - 'render_template', 'template'] - - -class RelationContext(dict): - """ - Base class for a context generator that gets relation data from juju. - - Subclasses must provide the attributes `name`, which is the name of the - interface of interest, `interface`, which is the type of the interface of - interest, and `required_keys`, which is the set of keys required for the - relation to be considered complete. The data for all interfaces matching - the `name` attribute that are complete will used to populate the dictionary - values (see `get_data`, below). - - The generated context will be namespaced under the relation :attr:`name`, - to prevent potential naming conflicts. - - :param str name: Override the relation :attr:`name`, since it can vary from charm to charm - :param list additional_required_keys: Extend the list of :attr:`required_keys` - """ - name = None - interface = None - - def __init__(self, name=None, additional_required_keys=None): - if not hasattr(self, 'required_keys'): - self.required_keys = [] - - if name is not None: - self.name = name - if additional_required_keys: - self.required_keys.extend(additional_required_keys) - self.get_data() - - def __bool__(self): - """ - Returns True if all of the required_keys are available. - """ - return self.is_ready() - - __nonzero__ = __bool__ - - def __repr__(self): - return super(RelationContext, self).__repr__() - - def is_ready(self): - """ - Returns True if all of the `required_keys` are available from any units. - """ - ready = len(self.get(self.name, [])) > 0 - if not ready: - hookenv.log('Incomplete relation: {}'.format(self.__class__.__name__), hookenv.DEBUG) - return ready - - def _is_ready(self, unit_data): - """ - Helper method that tests a set of relation data and returns True if - all of the `required_keys` are present. - """ - return set(unit_data.keys()).issuperset(set(self.required_keys)) - - def get_data(self): - """ - Retrieve the relation data for each unit involved in a relation and, - if complete, store it in a list under `self[self.name]`. This - is automatically called when the RelationContext is instantiated. - - The units are sorted lexographically first by the service ID, then by - the unit ID. Thus, if an interface has two other services, 'db:1' - and 'db:2', with 'db:1' having two units, 'wordpress/0' and 'wordpress/1', - and 'db:2' having one unit, 'mediawiki/0', all of which have a complete - set of data, the relation data for the units will be stored in the - order: 'wordpress/0', 'wordpress/1', 'mediawiki/0'. - - If you only care about a single unit on the relation, you can just - access it as `{{ interface[0]['key'] }}`. However, if you can at all - support multiple units on a relation, you should iterate over the list, - like:: - - {% for unit in interface -%} - {{ unit['key'] }}{% if not loop.last %},{% endif %} - {%- endfor %} - - Note that since all sets of relation data from all related services and - units are in a single list, if you need to know which service or unit a - set of data came from, you'll need to extend this class to preserve - that information. - """ - if not hookenv.relation_ids(self.name): - return - - ns = self.setdefault(self.name, []) - for rid in sorted(hookenv.relation_ids(self.name)): - for unit in sorted(hookenv.related_units(rid)): - reldata = hookenv.relation_get(rid=rid, unit=unit) - if self._is_ready(reldata): - ns.append(reldata) - - def provide_data(self): - """ - Return data to be relation_set for this interface. - """ - return {} - - -class MysqlRelation(RelationContext): - """ - Relation context for the `mysql` interface. - - :param str name: Override the relation :attr:`name`, since it can vary from charm to charm - :param list additional_required_keys: Extend the list of :attr:`required_keys` - """ - name = 'db' - interface = 'mysql' - - def __init__(self, *args, **kwargs): - self.required_keys = ['host', 'user', 'password', 'database'] - RelationContext.__init__(self, *args, **kwargs) - - -class HttpRelation(RelationContext): - """ - Relation context for the `http` interface. - - :param str name: Override the relation :attr:`name`, since it can vary from charm to charm - :param list additional_required_keys: Extend the list of :attr:`required_keys` - """ - name = 'website' - interface = 'http' - - def __init__(self, *args, **kwargs): - self.required_keys = ['host', 'port'] - RelationContext.__init__(self, *args, **kwargs) - - def provide_data(self): - return { - 'host': hookenv.unit_get('private-address'), - 'port': 80, - } - - -class RequiredConfig(dict): - """ - Data context that loads config options with one or more mandatory options. - - Once the required options have been changed from their default values, all - config options will be available, namespaced under `config` to prevent - potential naming conflicts (for example, between a config option and a - relation property). - - :param list *args: List of options that must be changed from their default values. - """ - - def __init__(self, *args): - self.required_options = args - self['config'] = hookenv.config() - with open(os.path.join(hookenv.charm_dir(), 'config.yaml')) as fp: - self.config = yaml.load(fp).get('options', {}) - - def __bool__(self): - for option in self.required_options: - if option not in self['config']: - return False - current_value = self['config'][option] - default_value = self.config[option].get('default') - if current_value == default_value: - return False - if current_value in (None, '') and default_value in (None, ''): - return False - return True - - def __nonzero__(self): - return self.__bool__() - - -class StoredContext(dict): - """ - A data context that always returns the data that it was first created with. - - This is useful to do a one-time generation of things like passwords, that - will thereafter use the same value that was originally generated, instead - of generating a new value each time it is run. - """ - def __init__(self, file_name, config_data): - """ - If the file exists, populate `self` with the data from the file. - Otherwise, populate with the given data and persist it to the file. - """ - if os.path.exists(file_name): - self.update(self.read_context(file_name)) - else: - self.store_context(file_name, config_data) - self.update(config_data) - - def store_context(self, file_name, config_data): - if not os.path.isabs(file_name): - file_name = os.path.join(hookenv.charm_dir(), file_name) - with open(file_name, 'w') as file_stream: - os.fchmod(file_stream.fileno(), 0o600) - yaml.dump(config_data, file_stream) - - def read_context(self, file_name): - if not os.path.isabs(file_name): - file_name = os.path.join(hookenv.charm_dir(), file_name) - with open(file_name, 'r') as file_stream: - data = yaml.load(file_stream) - if not data: - raise OSError("%s is empty" % file_name) - return data - - -class TemplateCallback(ManagerCallback): - """ - Callback class that will render a Jinja2 template, for use as a ready - action. - - :param str source: The template source file, relative to - `$CHARM_DIR/templates` - - :param str target: The target to write the rendered template to (or None) - :param str owner: The owner of the rendered file - :param str group: The group of the rendered file - :param int perms: The permissions of the rendered file - :param partial on_change_action: functools partial to be executed when - rendered file changes - :param jinja2 loader template_loader: A jinja2 template loader - - :return str: The rendered template - """ - def __init__(self, source, target, - owner='root', group='root', perms=0o444, - on_change_action=None, template_loader=None): - self.source = source - self.target = target - self.owner = owner - self.group = group - self.perms = perms - self.on_change_action = on_change_action - self.template_loader = template_loader - - def __call__(self, manager, service_name, event_name): - pre_checksum = '' - if self.on_change_action and os.path.isfile(self.target): - pre_checksum = host.file_hash(self.target) - service = manager.get_service(service_name) - context = {'ctx': {}} - for ctx in service.get('required_data', []): - context.update(ctx) - context['ctx'].update(ctx) - - result = templating.render(self.source, self.target, context, - self.owner, self.group, self.perms, - template_loader=self.template_loader) - if self.on_change_action: - if pre_checksum == host.file_hash(self.target): - hookenv.log( - 'No change detected: {}'.format(self.target), - hookenv.DEBUG) - else: - self.on_change_action() - - return result - - -# Convenience aliases for templates -render_template = template = TemplateCallback diff --git a/tests/charmhelpers/core/strutils.py b/tests/charmhelpers/core/strutils.py deleted file mode 100644 index e8df045..0000000 --- a/tests/charmhelpers/core/strutils.py +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import six -import re - - -def bool_from_string(value): - """Interpret string value as boolean. - - Returns True if value translates to True otherwise False. - """ - if isinstance(value, six.string_types): - value = six.text_type(value) - else: - msg = "Unable to interpret non-string value '%s' as boolean" % (value) - raise ValueError(msg) - - value = value.strip().lower() - - if value in ['y', 'yes', 'true', 't', 'on']: - return True - elif value in ['n', 'no', 'false', 'f', 'off']: - return False - - msg = "Unable to interpret string value '%s' as boolean" % (value) - raise ValueError(msg) - - -def bytes_from_string(value): - """Interpret human readable string value as bytes. - - Returns int - """ - BYTE_POWER = { - 'K': 1, - 'KB': 1, - 'M': 2, - 'MB': 2, - 'G': 3, - 'GB': 3, - 'T': 4, - 'TB': 4, - 'P': 5, - 'PB': 5, - } - if isinstance(value, six.string_types): - value = six.text_type(value) - else: - msg = "Unable to interpret non-string value '%s' as bytes" % (value) - raise ValueError(msg) - matches = re.match("([0-9]+)([a-zA-Z]+)", value) - if matches: - size = int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)]) - else: - # Assume that value passed in is bytes - try: - size = int(value) - except ValueError: - msg = "Unable to interpret string value '%s' as bytes" % (value) - raise ValueError(msg) - return size - - -class BasicStringComparator(object): - """Provides a class that will compare strings from an iterator type object. - Used to provide > and < comparisons on strings that may not necessarily be - alphanumerically ordered. e.g. OpenStack or Ubuntu releases AFTER the - z-wrap. - """ - - _list = None - - def __init__(self, item): - if self._list is None: - raise Exception("Must define the _list in the class definition!") - try: - self.index = self._list.index(item) - except Exception: - raise KeyError("Item '{}' is not in list '{}'" - .format(item, self._list)) - - def __eq__(self, other): - assert isinstance(other, str) or isinstance(other, self.__class__) - return self.index == self._list.index(other) - - def __ne__(self, other): - return not self.__eq__(other) - - def __lt__(self, other): - assert isinstance(other, str) or isinstance(other, self.__class__) - return self.index < self._list.index(other) - - def __ge__(self, other): - return not self.__lt__(other) - - def __gt__(self, other): - assert isinstance(other, str) or isinstance(other, self.__class__) - return self.index > self._list.index(other) - - def __le__(self, other): - return not self.__gt__(other) - - def __str__(self): - """Always give back the item at the index so it can be used in - comparisons like: - - s_mitaka = CompareOpenStack('mitaka') - s_newton = CompareOpenstack('newton') - - assert s_newton > s_mitaka - - @returns: - """ - return self._list[self.index] diff --git a/tests/charmhelpers/core/sysctl.py b/tests/charmhelpers/core/sysctl.py deleted file mode 100644 index 6e413e3..0000000 --- a/tests/charmhelpers/core/sysctl.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import yaml - -from subprocess import check_call - -from charmhelpers.core.hookenv import ( - log, - DEBUG, - ERROR, -) - -__author__ = 'Jorge Niedbalski R. ' - - -def create(sysctl_dict, sysctl_file): - """Creates a sysctl.conf file from a YAML associative array - - :param sysctl_dict: a YAML-formatted string of sysctl options eg "{ 'kernel.max_pid': 1337 }" - :type sysctl_dict: str - :param sysctl_file: path to the sysctl file to be saved - :type sysctl_file: str or unicode - :returns: None - """ - try: - sysctl_dict_parsed = yaml.safe_load(sysctl_dict) - except yaml.YAMLError: - log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict), - level=ERROR) - return - - with open(sysctl_file, "w") as fd: - for key, value in sysctl_dict_parsed.items(): - fd.write("{}={}\n".format(key, value)) - - log("Updating sysctl_file: %s values: %s" % (sysctl_file, sysctl_dict_parsed), - level=DEBUG) - - check_call(["sysctl", "-p", sysctl_file]) diff --git a/tests/charmhelpers/core/templating.py b/tests/charmhelpers/core/templating.py deleted file mode 100644 index 7b801a3..0000000 --- a/tests/charmhelpers/core/templating.py +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys - -from charmhelpers.core import host -from charmhelpers.core import hookenv - - -def render(source, target, context, owner='root', group='root', - perms=0o444, templates_dir=None, encoding='UTF-8', template_loader=None): - """ - Render a template. - - The `source` path, if not absolute, is relative to the `templates_dir`. - - The `target` path should be absolute. It can also be `None`, in which - case no file will be written. - - The context should be a dict containing the values to be replaced in the - template. - - The `owner`, `group`, and `perms` options will be passed to `write_file`. - - If omitted, `templates_dir` defaults to the `templates` folder in the charm. - - The rendered template will be written to the file as well as being returned - as a string. - - Note: Using this requires python-jinja2 or python3-jinja2; if it is not - installed, calling this will attempt to use charmhelpers.fetch.apt_install - to install it. - """ - try: - from jinja2 import FileSystemLoader, Environment, exceptions - except ImportError: - try: - from charmhelpers.fetch import apt_install - except ImportError: - hookenv.log('Could not import jinja2, and could not import ' - 'charmhelpers.fetch to install it', - level=hookenv.ERROR) - raise - if sys.version_info.major == 2: - apt_install('python-jinja2', fatal=True) - else: - apt_install('python3-jinja2', fatal=True) - from jinja2 import FileSystemLoader, Environment, exceptions - - if template_loader: - template_env = Environment(loader=template_loader) - else: - if templates_dir is None: - templates_dir = os.path.join(hookenv.charm_dir(), 'templates') - template_env = Environment(loader=FileSystemLoader(templates_dir)) - try: - source = source - template = template_env.get_template(source) - except exceptions.TemplateNotFound as e: - hookenv.log('Could not load template %s from %s.' % - (source, templates_dir), - level=hookenv.ERROR) - raise e - content = template.render(context) - if target is not None: - target_dir = os.path.dirname(target) - if not os.path.exists(target_dir): - # This is a terrible default directory permission, as the file - # or its siblings will often contain secrets. - host.mkdir(os.path.dirname(target), owner, group, perms=0o755) - host.write_file(target, content.encode(encoding), owner, group, perms) - return content diff --git a/tests/charmhelpers/core/unitdata.py b/tests/charmhelpers/core/unitdata.py deleted file mode 100644 index 7af875c..0000000 --- a/tests/charmhelpers/core/unitdata.py +++ /dev/null @@ -1,518 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Authors: -# Kapil Thangavelu -# -""" -Intro ------ - -A simple way to store state in units. This provides a key value -storage with support for versioned, transactional operation, -and can calculate deltas from previous values to simplify unit logic -when processing changes. - - -Hook Integration ----------------- - -There are several extant frameworks for hook execution, including - - - charmhelpers.core.hookenv.Hooks - - charmhelpers.core.services.ServiceManager - -The storage classes are framework agnostic, one simple integration is -via the HookData contextmanager. It will record the current hook -execution environment (including relation data, config data, etc.), -setup a transaction and allow easy access to the changes from -previously seen values. One consequence of the integration is the -reservation of particular keys ('rels', 'unit', 'env', 'config', -'charm_revisions') for their respective values. - -Here's a fully worked integration example using hookenv.Hooks:: - - from charmhelper.core import hookenv, unitdata - - hook_data = unitdata.HookData() - db = unitdata.kv() - hooks = hookenv.Hooks() - - @hooks.hook - def config_changed(): - # Print all changes to configuration from previously seen - # values. - for changed, (prev, cur) in hook_data.conf.items(): - print('config changed', changed, - 'previous value', prev, - 'current value', cur) - - # Get some unit specific bookeeping - if not db.get('pkg_key'): - key = urllib.urlopen('https://example.com/pkg_key').read() - db.set('pkg_key', key) - - # Directly access all charm config as a mapping. - conf = db.getrange('config', True) - - # Directly access all relation data as a mapping - rels = db.getrange('rels', True) - - if __name__ == '__main__': - with hook_data(): - hook.execute() - - -A more basic integration is via the hook_scope context manager which simply -manages transaction scope (and records hook name, and timestamp):: - - >>> from unitdata import kv - >>> db = kv() - >>> with db.hook_scope('install'): - ... # do work, in transactional scope. - ... db.set('x', 1) - >>> db.get('x') - 1 - - -Usage ------ - -Values are automatically json de/serialized to preserve basic typing -and complex data struct capabilities (dicts, lists, ints, booleans, etc). - -Individual values can be manipulated via get/set:: - - >>> kv.set('y', True) - >>> kv.get('y') - True - - # We can set complex values (dicts, lists) as a single key. - >>> kv.set('config', {'a': 1, 'b': True'}) - - # Also supports returning dictionaries as a record which - # provides attribute access. - >>> config = kv.get('config', record=True) - >>> config.b - True - - -Groups of keys can be manipulated with update/getrange:: - - >>> kv.update({'z': 1, 'y': 2}, prefix="gui.") - >>> kv.getrange('gui.', strip=True) - {'z': 1, 'y': 2} - -When updating values, its very helpful to understand which values -have actually changed and how have they changed. The storage -provides a delta method to provide for this:: - - >>> data = {'debug': True, 'option': 2} - >>> delta = kv.delta(data, 'config.') - >>> delta.debug.previous - None - >>> delta.debug.current - True - >>> delta - {'debug': (None, True), 'option': (None, 2)} - -Note the delta method does not persist the actual change, it needs to -be explicitly saved via 'update' method:: - - >>> kv.update(data, 'config.') - -Values modified in the context of a hook scope retain historical values -associated to the hookname. - - >>> with db.hook_scope('config-changed'): - ... db.set('x', 42) - >>> db.gethistory('x') - [(1, u'x', 1, u'install', u'2015-01-21T16:49:30.038372'), - (2, u'x', 42, u'config-changed', u'2015-01-21T16:49:30.038786')] - -""" - -import collections -import contextlib -import datetime -import itertools -import json -import os -import pprint -import sqlite3 -import sys - -__author__ = 'Kapil Thangavelu ' - - -class Storage(object): - """Simple key value database for local unit state within charms. - - Modifications are not persisted unless :meth:`flush` is called. - - To support dicts, lists, integer, floats, and booleans values - are automatically json encoded/decoded. - """ - def __init__(self, path=None): - self.db_path = path - if path is None: - if 'UNIT_STATE_DB' in os.environ: - self.db_path = os.environ['UNIT_STATE_DB'] - else: - self.db_path = os.path.join( - os.environ.get('CHARM_DIR', ''), '.unit-state.db') - self.conn = sqlite3.connect('%s' % self.db_path) - self.cursor = self.conn.cursor() - self.revision = None - self._closed = False - self._init() - - def close(self): - if self._closed: - return - self.flush(False) - self.cursor.close() - self.conn.close() - self._closed = True - - def get(self, key, default=None, record=False): - self.cursor.execute('select data from kv where key=?', [key]) - result = self.cursor.fetchone() - if not result: - return default - if record: - return Record(json.loads(result[0])) - return json.loads(result[0]) - - def getrange(self, key_prefix, strip=False): - """ - Get a range of keys starting with a common prefix as a mapping of - keys to values. - - :param str key_prefix: Common prefix among all keys - :param bool strip: Optionally strip the common prefix from the key - names in the returned dict - :return dict: A (possibly empty) dict of key-value mappings - """ - self.cursor.execute("select key, data from kv where key like ?", - ['%s%%' % key_prefix]) - result = self.cursor.fetchall() - - if not result: - return {} - if not strip: - key_prefix = '' - return dict([ - (k[len(key_prefix):], json.loads(v)) for k, v in result]) - - def update(self, mapping, prefix=""): - """ - Set the values of multiple keys at once. - - :param dict mapping: Mapping of keys to values - :param str prefix: Optional prefix to apply to all keys in `mapping` - before setting - """ - for k, v in mapping.items(): - self.set("%s%s" % (prefix, k), v) - - def unset(self, key): - """ - Remove a key from the database entirely. - """ - self.cursor.execute('delete from kv where key=?', [key]) - if self.revision and self.cursor.rowcount: - self.cursor.execute( - 'insert into kv_revisions values (?, ?, ?)', - [key, self.revision, json.dumps('DELETED')]) - - def unsetrange(self, keys=None, prefix=""): - """ - Remove a range of keys starting with a common prefix, from the database - entirely. - - :param list keys: List of keys to remove. - :param str prefix: Optional prefix to apply to all keys in ``keys`` - before removing. - """ - if keys is not None: - keys = ['%s%s' % (prefix, key) for key in keys] - self.cursor.execute('delete from kv where key in (%s)' % ','.join(['?'] * len(keys)), keys) - if self.revision and self.cursor.rowcount: - self.cursor.execute( - 'insert into kv_revisions values %s' % ','.join(['(?, ?, ?)'] * len(keys)), - list(itertools.chain.from_iterable((key, self.revision, json.dumps('DELETED')) for key in keys))) - else: - self.cursor.execute('delete from kv where key like ?', - ['%s%%' % prefix]) - if self.revision and self.cursor.rowcount: - self.cursor.execute( - 'insert into kv_revisions values (?, ?, ?)', - ['%s%%' % prefix, self.revision, json.dumps('DELETED')]) - - def set(self, key, value): - """ - Set a value in the database. - - :param str key: Key to set the value for - :param value: Any JSON-serializable value to be set - """ - serialized = json.dumps(value) - - self.cursor.execute('select data from kv where key=?', [key]) - exists = self.cursor.fetchone() - - # Skip mutations to the same value - if exists: - if exists[0] == serialized: - return value - - if not exists: - self.cursor.execute( - 'insert into kv (key, data) values (?, ?)', - (key, serialized)) - else: - self.cursor.execute(''' - update kv - set data = ? - where key = ?''', [serialized, key]) - - # Save - if not self.revision: - return value - - self.cursor.execute( - 'select 1 from kv_revisions where key=? and revision=?', - [key, self.revision]) - exists = self.cursor.fetchone() - - if not exists: - self.cursor.execute( - '''insert into kv_revisions ( - revision, key, data) values (?, ?, ?)''', - (self.revision, key, serialized)) - else: - self.cursor.execute( - ''' - update kv_revisions - set data = ? - where key = ? - and revision = ?''', - [serialized, key, self.revision]) - - return value - - def delta(self, mapping, prefix): - """ - return a delta containing values that have changed. - """ - previous = self.getrange(prefix, strip=True) - if not previous: - pk = set() - else: - pk = set(previous.keys()) - ck = set(mapping.keys()) - delta = DeltaSet() - - # added - for k in ck.difference(pk): - delta[k] = Delta(None, mapping[k]) - - # removed - for k in pk.difference(ck): - delta[k] = Delta(previous[k], None) - - # changed - for k in pk.intersection(ck): - c = mapping[k] - p = previous[k] - if c != p: - delta[k] = Delta(p, c) - - return delta - - @contextlib.contextmanager - def hook_scope(self, name=""): - """Scope all future interactions to the current hook execution - revision.""" - assert not self.revision - self.cursor.execute( - 'insert into hooks (hook, date) values (?, ?)', - (name or sys.argv[0], - datetime.datetime.utcnow().isoformat())) - self.revision = self.cursor.lastrowid - try: - yield self.revision - self.revision = None - except Exception: - self.flush(False) - self.revision = None - raise - else: - self.flush() - - def flush(self, save=True): - if save: - self.conn.commit() - elif self._closed: - return - else: - self.conn.rollback() - - def _init(self): - self.cursor.execute(''' - create table if not exists kv ( - key text, - data text, - primary key (key) - )''') - self.cursor.execute(''' - create table if not exists kv_revisions ( - key text, - revision integer, - data text, - primary key (key, revision) - )''') - self.cursor.execute(''' - create table if not exists hooks ( - version integer primary key autoincrement, - hook text, - date text - )''') - self.conn.commit() - - def gethistory(self, key, deserialize=False): - self.cursor.execute( - ''' - select kv.revision, kv.key, kv.data, h.hook, h.date - from kv_revisions kv, - hooks h - where kv.key=? - and kv.revision = h.version - ''', [key]) - if deserialize is False: - return self.cursor.fetchall() - return map(_parse_history, self.cursor.fetchall()) - - def debug(self, fh=sys.stderr): - self.cursor.execute('select * from kv') - pprint.pprint(self.cursor.fetchall(), stream=fh) - self.cursor.execute('select * from kv_revisions') - pprint.pprint(self.cursor.fetchall(), stream=fh) - - -def _parse_history(d): - return (d[0], d[1], json.loads(d[2]), d[3], - datetime.datetime.strptime(d[-1], "%Y-%m-%dT%H:%M:%S.%f")) - - -class HookData(object): - """Simple integration for existing hook exec frameworks. - - Records all unit information, and stores deltas for processing - by the hook. - - Sample:: - - from charmhelper.core import hookenv, unitdata - - changes = unitdata.HookData() - db = unitdata.kv() - hooks = hookenv.Hooks() - - @hooks.hook - def config_changed(): - # View all changes to configuration - for changed, (prev, cur) in changes.conf.items(): - print('config changed', changed, - 'previous value', prev, - 'current value', cur) - - # Get some unit specific bookeeping - if not db.get('pkg_key'): - key = urllib.urlopen('https://example.com/pkg_key').read() - db.set('pkg_key', key) - - if __name__ == '__main__': - with changes(): - hook.execute() - - """ - def __init__(self): - self.kv = kv() - self.conf = None - self.rels = None - - @contextlib.contextmanager - def __call__(self): - from charmhelpers.core import hookenv - hook_name = hookenv.hook_name() - - with self.kv.hook_scope(hook_name): - self._record_charm_version(hookenv.charm_dir()) - delta_config, delta_relation = self._record_hook(hookenv) - yield self.kv, delta_config, delta_relation - - def _record_charm_version(self, charm_dir): - # Record revisions.. charm revisions are meaningless - # to charm authors as they don't control the revision. - # so logic dependnent on revision is not particularly - # useful, however it is useful for debugging analysis. - charm_rev = open( - os.path.join(charm_dir, 'revision')).read().strip() - charm_rev = charm_rev or '0' - revs = self.kv.get('charm_revisions', []) - if charm_rev not in revs: - revs.append(charm_rev.strip() or '0') - self.kv.set('charm_revisions', revs) - - def _record_hook(self, hookenv): - data = hookenv.execution_environment() - self.conf = conf_delta = self.kv.delta(data['conf'], 'config') - self.rels = rels_delta = self.kv.delta(data['rels'], 'rels') - self.kv.set('env', dict(data['env'])) - self.kv.set('unit', data['unit']) - self.kv.set('relid', data.get('relid')) - return conf_delta, rels_delta - - -class Record(dict): - - __slots__ = () - - def __getattr__(self, k): - if k in self: - return self[k] - raise AttributeError(k) - - -class DeltaSet(Record): - - __slots__ = () - - -Delta = collections.namedtuple('Delta', ['previous', 'current']) - - -_KV = None - - -def kv(): - global _KV - if _KV is None: - _KV = Storage() - return _KV diff --git a/tests/charmhelpers/osplatform.py b/tests/charmhelpers/osplatform.py deleted file mode 100644 index d9a4d5c..0000000 --- a/tests/charmhelpers/osplatform.py +++ /dev/null @@ -1,25 +0,0 @@ -import platform - - -def get_platform(): - """Return the current OS platform. - - For example: if current os platform is Ubuntu then a string "ubuntu" - will be returned (which is the name of the module). - This string is used to decide which platform module should be imported. - """ - # linux_distribution is deprecated and will be removed in Python 3.7 - # Warings *not* disabled, as we certainly need to fix this. - tuple_platform = platform.linux_distribution() - current_platform = tuple_platform[0] - if "Ubuntu" in current_platform: - return "ubuntu" - elif "CentOS" in current_platform: - return "centos" - elif "debian" in current_platform: - # Stock Python does not detect Ubuntu and instead returns debian. - # Or at least it does in some build environments like Travis CI - return "ubuntu" - else: - raise RuntimeError("This module is not supported on {}." - .format(current_platform)) diff --git a/tests/gate-basic-artful-pike b/tests/gate-basic-artful-pike deleted file mode 100644 index 50bd95e..0000000 --- a/tests/gate-basic-artful-pike +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Amulet tests on a basic ceph deployment on artful-pike.""" - -from basic_deployment import CephBasicDeployment - -if __name__ == '__main__': - deployment = CephBasicDeployment(series='artful') - deployment.run_tests() - -# NOTE(beisner): Artful target disabled, pending bug: -# https://bugs.launchpad.net/charm-percona-cluster/+bug/1728132 diff --git a/tests/gate-basic-trusty-icehouse b/tests/gate-basic-trusty-icehouse deleted file mode 100755 index 8a98793..0000000 --- a/tests/gate-basic-trusty-icehouse +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Amulet tests on a basic ceph deployment on trusty-icehouse.""" - -from basic_deployment import CephBasicDeployment - -if __name__ == '__main__': - deployment = CephBasicDeployment(series='trusty') - deployment.run_tests() diff --git a/tests/gate-basic-trusty-kilo b/tests/gate-basic-trusty-kilo deleted file mode 100755 index 86e772a..0000000 --- a/tests/gate-basic-trusty-kilo +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Amulet tests on a basic ceph deployment on trusty-kilo.""" - -from basic_deployment import CephBasicDeployment - -if __name__ == '__main__': - deployment = CephBasicDeployment(series='trusty', - openstack='cloud:trusty-kilo', - source='cloud:trusty-updates/kilo') - deployment.run_tests() diff --git a/tests/gate-basic-trusty-liberty b/tests/gate-basic-trusty-liberty deleted file mode 100755 index 3dfa8b6..0000000 --- a/tests/gate-basic-trusty-liberty +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Amulet tests on a basic ceph deployment on trusty-liberty.""" - -from basic_deployment import CephBasicDeployment - -if __name__ == '__main__': - deployment = CephBasicDeployment(series='trusty', - openstack='cloud:trusty-liberty', - source='cloud:trusty-updates/liberty') - deployment.run_tests() diff --git a/tests/gate-basic-trusty-mitaka b/tests/gate-basic-trusty-mitaka deleted file mode 100755 index 52b688f..0000000 --- a/tests/gate-basic-trusty-mitaka +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Amulet tests on a basic ceph deployment on trusty-mitaka.""" - -from basic_deployment import CephBasicDeployment - -if __name__ == '__main__': - deployment = CephBasicDeployment(series='trusty', - openstack='cloud:trusty-mitaka', - source='cloud:trusty-updates/mitaka') - deployment.run_tests() diff --git a/tests/gate-basic-xenial-mitaka b/tests/gate-basic-xenial-mitaka deleted file mode 100755 index f897700..0000000 --- a/tests/gate-basic-xenial-mitaka +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Amulet tests on a basic ceph deployment on xenial-mitaka.""" - -from basic_deployment import CephBasicDeployment - -if __name__ == '__main__': - deployment = CephBasicDeployment(series='xenial') - deployment.run_tests() diff --git a/tests/gate-basic-xenial-newton b/tests/gate-basic-xenial-newton deleted file mode 100755 index 69bf0a5..0000000 --- a/tests/gate-basic-xenial-newton +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Amulet tests on a basic ceph deployment on xenial-newton.""" - -from basic_deployment import CephBasicDeployment - -if __name__ == '__main__': - deployment = CephBasicDeployment(series='xenial', - openstack='cloud:xenial-newton', - source='cloud:xenial-updates/newton') - deployment.run_tests() diff --git a/tests/gate-basic-xenial-ocata b/tests/gate-basic-xenial-ocata deleted file mode 100755 index ec2713c..0000000 --- a/tests/gate-basic-xenial-ocata +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Amulet tests on a basic ceph deployment on xenial-ocata.""" - -from basic_deployment import CephBasicDeployment - -if __name__ == '__main__': - deployment = CephBasicDeployment(series='xenial', - openstack='cloud:xenial-ocata', - source='cloud:xenial-updates/ocata') - deployment.run_tests() diff --git a/tests/gate-basic-xenial-pike b/tests/gate-basic-xenial-pike deleted file mode 100755 index 2fafe3b..0000000 --- a/tests/gate-basic-xenial-pike +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Amulet tests on a basic ceph deployment on xenial-pike.""" - -from basic_deployment import CephBasicDeployment - -if __name__ == '__main__': - deployment = CephBasicDeployment(series='xenial', - openstack='cloud:xenial-pike', - source='cloud:xenial-updates/pike') - deployment.run_tests() diff --git a/tests/gate-basic-zesty-ocata b/tests/gate-basic-zesty-ocata deleted file mode 100755 index a642193..0000000 --- a/tests/gate-basic-zesty-ocata +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Amulet tests on a basic ceph deployment on zesty-ocata.""" - -from basic_deployment import CephBasicDeployment - -if __name__ == '__main__': - deployment = CephBasicDeployment(series='zesty') - deployment.run_tests() diff --git a/tests/tests.yaml b/tests/tests.yaml deleted file mode 100644 index a03e7ba..0000000 --- a/tests/tests.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Bootstrap the model if necessary. -bootstrap: True -# Re-use bootstrap node. -reset: True -# Use tox/requirements to drive the venv instead of bundletester's venv feature. -virtualenv: False -# Leave makefile empty, otherwise unit/lint tests will rerun ahead of amulet. -makefile: [] -# Do not specify juju PPA sources. Juju is presumed to be pre-installed -# and configured in all test runner environments. -#sources: -# Do not specify or rely on system packages. -#packages: -# Do not specify python packages here. Use test-requirements.txt -# and tox instead. ie. The venv is constructed before bundletester -# is invoked. -#python-packages: -reset_timeout: 600 diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 6d44f4b..0000000 --- a/tox.ini +++ /dev/null @@ -1,85 +0,0 @@ -# Classic charm: ./tox.ini -# This file is managed centrally by release-tools and should not be modified -# within individual charm repos. -[tox] -envlist = pep8,py27 -skipsdist = True - -[testenv] -setenv = VIRTUAL_ENV={envdir} - PYTHONHASHSEED=0 - CHARM_DIR={envdir} - AMULET_SETUP_TIMEOUT=2700 -install_command = - pip install --allow-unverified python-apt {opts} {packages} -commands = ostestr {posargs} -whitelist_externals = juju -passenv = HOME TERM AMULET_* CS_API_* - -[testenv:py27] -basepython = python2.7 -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt - -[testenv:py35] -basepython = python3.5 -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt - -[testenv:pep8] -basepython = python2.7 -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt -commands = flake8 {posargs} hooks unit_tests tests actions lib - charm-proof - -[testenv:venv] -commands = {posargs} - -[testenv:func27-noop] -# DRY RUN - For Debug -basepython = python2.7 -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt -commands = - bundletester -vl DEBUG -r json -o func-results.json --test-pattern "gate-*" -n --no-destroy - -[testenv:func27] -# Charm Functional Test -# Run all gate tests which are +x (expected to always pass) -basepython = python2.7 -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt -commands = - bundletester -vl DEBUG -r json -o func-results.json --test-pattern "gate-*" --no-destroy - -[testenv:func27-smoke] -# Charm Functional Test -# Run a specific test as an Amulet smoke test (expected to always pass) -basepython = python2.7 -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt -commands = - bundletester -vl DEBUG -r json -o func-results.json gate-basic-xenial-pike --no-destroy - -[testenv:func27-dfs] -# Charm Functional Test -# Run all deploy-from-source tests which are +x (may not always pass!) -basepython = python2.7 -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt -commands = - bundletester -vl DEBUG -r json -o func-results.json --test-pattern "dfs-*" --no-destroy - -[testenv:func27-dev] -# Charm Functional Test -# Run all development test targets which are +x (may not always pass!) -basepython = python2.7 -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt -commands = - bundletester -vl DEBUG -r json -o func-results.json --test-pattern "dev-*" --no-destroy - -[flake8] -ignore = E402,E226 -exclude = */charmhelpers diff --git a/unit_tests/__init__.py b/unit_tests/__init__.py deleted file mode 100644 index 84f643d..0000000 --- a/unit_tests/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -sys.path.append('hooks') -sys.path.append('lib') -sys.path.append('actions') diff --git a/unit_tests/test_actions_pause_resume.py b/unit_tests/test_actions_pause_resume.py deleted file mode 100644 index 8cf8fe7..0000000 --- a/unit_tests/test_actions_pause_resume.py +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import mock - -import sys - -from test_utils import CharmTestCase - -# python-apt is not installed as part of test-requirements but is imported by -# some charmhelpers modules so create a fake import. -mock_apt = mock.MagicMock() -sys.modules['apt'] = mock_apt -mock_apt.apt_pkg = mock.MagicMock() - -with mock.patch('charmhelpers.contrib.hardening.harden.harden') as mock_dec: - mock_dec.side_effect = (lambda *dargs, **dkwargs: lambda f: - lambda *args, **kwargs: f(*args, **kwargs)) - # import ceph_hooks as hooks - import pause_resume as actions - - -class PauseTestCase(CharmTestCase): - - def setUp(self): - super(PauseTestCase, self).setUp( - actions, ["check_call", - "get_local_osd_ids", - "set_unit_paused", - "assess_status"]) - - def test_pauses_services(self): - self.get_local_osd_ids.return_value = [5] - actions.pause([]) - cmd = ['ceph', '--id', - 'osd-upgrade', 'osd', 'out', '5'] - self.check_call.assert_called_once_with(cmd) - self.set_unit_paused.assert_called_once_with() - self.assess_status.assert_called_once_with() - - -class ResumeTestCase(CharmTestCase): - - def setUp(self): - super(ResumeTestCase, self).setUp( - actions, ["check_call", - "get_local_osd_ids", - "clear_unit_paused", - "assess_status"]) - - def test_pauses_services(self): - self.get_local_osd_ids.return_value = [5] - actions.resume([]) - cmd = ['ceph', '--id', - 'osd-upgrade', 'osd', 'in', '5'] - self.check_call.assert_called_once_with(cmd) - self.clear_unit_paused.assert_called_once_with() - self.assess_status.assert_called_once_with() - - -class MainTestCase(CharmTestCase): - - def setUp(self): - super(MainTestCase, self).setUp(actions, ["action_fail"]) - - def test_invokes_action(self): - dummy_calls = [] - - def dummy_action(args): - dummy_calls.append(True) - - with mock.patch.dict(actions.ACTIONS, {"foo": dummy_action}): - actions.main(["foo"]) - self.assertEqual(dummy_calls, [True]) - - def test_unknown_action(self): - """Unknown actions aren't a traceback.""" - exit_string = actions.main(["foo"]) - self.assertEqual("Action foo undefined", exit_string) - - def test_failing_action(self): - """Actions which traceback trigger action_fail() calls.""" - dummy_calls = [] - - self.action_fail.side_effect = dummy_calls.append - - def dummy_action(args): - raise ValueError("uh oh") - - with mock.patch.dict(actions.ACTIONS, {"foo": dummy_action}): - actions.main(["foo"]) - self.assertEqual(dummy_calls, ["Action foo failed: uh oh"]) diff --git a/unit_tests/test_ceph_hooks.py b/unit_tests/test_ceph_hooks.py deleted file mode 100644 index b7439e0..0000000 --- a/unit_tests/test_ceph_hooks.py +++ /dev/null @@ -1,333 +0,0 @@ -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import copy -import unittest - -from mock import patch, DEFAULT, call - -import charmhelpers.contrib.storage.linux.ceph as ceph -import ceph_hooks - - -CHARM_CONFIG = {'config-flags': '', - 'auth-supported': False, - 'fsid': '1234', - 'loglevel': 1, - 'use-syslog': True, - 'osd-journal-size': 1024, - 'use-direct-io': True, - 'osd-format': 'ext4', - 'prefer-ipv6': False, - 'customize-failure-domain': False, - 'bluestore': False, - 'default-rbd-features': None} - - -class CephHooksTestCase(unittest.TestCase): - - @patch.object(ceph_hooks, 'get_public_addr', lambda *args: "10.0.0.1") - @patch.object(ceph_hooks, 'get_cluster_addr', lambda *args: "10.1.0.1") - @patch.object(ceph_hooks, 'cmp_pkgrevno', lambda *args: 1) - @patch.object(ceph_hooks, 'get_mon_hosts', lambda *args: ['10.0.0.1', - '10.0.0.2']) - @patch.object(ceph_hooks, 'get_networks', lambda *args: "") - @patch.object(ceph, 'config') - @patch.object(ceph_hooks, 'config') - def test_get_ceph_context(self, mock_config, mock_config2): - config = copy.deepcopy(CHARM_CONFIG) - mock_config.side_effect = lambda key: config[key] - mock_config2.side_effect = lambda key: config[key] - ctxt = ceph_hooks.get_ceph_context() - expected = {'auth_supported': False, - 'ceph_cluster_network': '', - 'ceph_public_network': '', - 'cluster_addr': '10.1.0.1', - 'dio': 'true', - 'fsid': '1234', - 'loglevel': 1, - 'mon_hosts': '10.0.0.1 10.0.0.2', - 'old_auth': False, - 'osd_journal_size': 1024, - 'public_addr': '10.0.0.1', - 'short_object_len': True, - 'use_syslog': 'true', - 'bluestore': False, - 'bluestore_experimental': False} - self.assertEqual(ctxt, expected) - - @patch.object(ceph_hooks, 'get_public_addr', lambda *args: "10.0.0.1") - @patch.object(ceph_hooks, 'get_cluster_addr', lambda *args: "10.1.0.1") - @patch.object(ceph_hooks, 'cmp_pkgrevno', - lambda pkg, ver: -1 if ver == '12.1.0' else 1) - @patch.object(ceph_hooks, 'get_mon_hosts', lambda *args: ['10.0.0.1', - '10.0.0.2']) - @patch.object(ceph_hooks, 'get_networks', lambda *args: "") - @patch.object(ceph, 'config') - @patch.object(ceph_hooks, 'config') - def test_get_ceph_context_filestore_old(self, mock_config, mock_config2): - config = copy.deepcopy(CHARM_CONFIG) - mock_config.side_effect = lambda key: config[key] - mock_config2.side_effect = lambda key: config[key] - ctxt = ceph_hooks.get_ceph_context() - expected = {'auth_supported': False, - 'ceph_cluster_network': '', - 'ceph_public_network': '', - 'cluster_addr': '10.1.0.1', - 'dio': 'true', - 'fsid': '1234', - 'loglevel': 1, - 'mon_hosts': '10.0.0.1 10.0.0.2', - 'old_auth': False, - 'osd_journal_size': 1024, - 'public_addr': '10.0.0.1', - 'short_object_len': True, - 'use_syslog': 'true', - 'bluestore': False, - 'bluestore_experimental': True} - self.assertEqual(ctxt, expected) - - @patch.object(ceph_hooks, 'get_public_addr', lambda *args: "10.0.0.1") - @patch.object(ceph_hooks, 'get_cluster_addr', lambda *args: "10.1.0.1") - @patch.object(ceph_hooks, 'cmp_pkgrevno', lambda *args: 1) - @patch.object(ceph_hooks, 'get_mon_hosts', lambda *args: ['10.0.0.1', - '10.0.0.2']) - @patch.object(ceph_hooks, 'get_networks', lambda *args: "") - @patch.object(ceph, 'config') - @patch.object(ceph_hooks, 'config') - def test_get_ceph_context_bluestore(self, mock_config, mock_config2): - config = copy.deepcopy(CHARM_CONFIG) - config['bluestore'] = True - mock_config.side_effect = lambda key: config[key] - mock_config2.side_effect = lambda key: config[key] - ctxt = ceph_hooks.get_ceph_context() - expected = {'auth_supported': False, - 'ceph_cluster_network': '', - 'ceph_public_network': '', - 'cluster_addr': '10.1.0.1', - 'dio': 'true', - 'fsid': '1234', - 'loglevel': 1, - 'mon_hosts': '10.0.0.1 10.0.0.2', - 'old_auth': False, - 'osd_journal_size': 1024, - 'public_addr': '10.0.0.1', - 'short_object_len': True, - 'use_syslog': 'true', - 'bluestore': True, - 'bluestore_experimental': False} - self.assertEqual(ctxt, expected) - - @patch.object(ceph_hooks, 'get_public_addr', lambda *args: "10.0.0.1") - @patch.object(ceph_hooks, 'get_cluster_addr', lambda *args: "10.1.0.1") - @patch.object(ceph_hooks, 'cmp_pkgrevno', - lambda pkg, ver: -1 if ver == '12.1.0' else 1) - @patch.object(ceph_hooks, 'get_mon_hosts', lambda *args: ['10.0.0.1', - '10.0.0.2']) - @patch.object(ceph_hooks, 'get_networks', lambda *args: "") - @patch.object(ceph, 'config') - @patch.object(ceph_hooks, 'config') - def test_get_ceph_context_bluestore_old(self, mock_config, mock_config2): - config = copy.deepcopy(CHARM_CONFIG) - config['bluestore'] = True - mock_config.side_effect = lambda key: config[key] - mock_config2.side_effect = lambda key: config[key] - ctxt = ceph_hooks.get_ceph_context() - expected = {'auth_supported': False, - 'ceph_cluster_network': '', - 'ceph_public_network': '', - 'cluster_addr': '10.1.0.1', - 'dio': 'true', - 'fsid': '1234', - 'loglevel': 1, - 'mon_hosts': '10.0.0.1 10.0.0.2', - 'old_auth': False, - 'osd_journal_size': 1024, - 'public_addr': '10.0.0.1', - 'short_object_len': True, - 'use_syslog': 'true', - 'bluestore': True, - 'bluestore_experimental': True} - self.assertEqual(ctxt, expected) - - @patch.object(ceph_hooks, 'get_public_addr', lambda *args: "10.0.0.1") - @patch.object(ceph_hooks, 'get_cluster_addr', lambda *args: "10.1.0.1") - @patch.object(ceph_hooks, 'cmp_pkgrevno', - lambda pkg, ver: -1 if ver == '12.1.0' else 1) - @patch.object(ceph_hooks, 'get_mon_hosts', lambda *args: ['10.0.0.1', - '10.0.0.2']) - @patch.object(ceph_hooks, 'get_networks', lambda *args: "") - @patch.object(ceph, 'config') - @patch.object(ceph_hooks, 'config') - def test_get_ceph_context_rbd_features(self, mock_config, mock_config2): - config = copy.deepcopy(CHARM_CONFIG) - config['default-rbd-features'] = 1 - mock_config.side_effect = lambda key: config[key] - mock_config2.side_effect = lambda key: config[key] - ctxt = ceph_hooks.get_ceph_context() - expected = {'auth_supported': False, - 'ceph_cluster_network': '', - 'ceph_public_network': '', - 'cluster_addr': '10.1.0.1', - 'dio': 'true', - 'fsid': '1234', - 'loglevel': 1, - 'mon_hosts': '10.0.0.1 10.0.0.2', - 'old_auth': False, - 'osd_journal_size': 1024, - 'public_addr': '10.0.0.1', - 'short_object_len': True, - 'use_syslog': 'true', - 'rbd_features': 1, - 'bluestore': False, - 'bluestore_experimental': True} - self.assertEqual(ctxt, expected) - - @patch.object(ceph_hooks, 'get_public_addr', lambda *args: "10.0.0.1") - @patch.object(ceph_hooks, 'get_cluster_addr', lambda *args: "10.1.0.1") - @patch.object(ceph_hooks, 'cmp_pkgrevno', lambda *args: 1) - @patch.object(ceph_hooks, 'get_mon_hosts', lambda *args: ['10.0.0.1', - '10.0.0.2']) - @patch.object(ceph_hooks, 'get_networks', lambda *args: "") - @patch.object(ceph, 'config') - @patch.object(ceph_hooks, 'config') - def test_get_ceph_context_w_config_flags(self, mock_config, mock_config2): - config = copy.deepcopy(CHARM_CONFIG) - config['config-flags'] = '{"osd": {"osd max write size": 1024}}' - mock_config.side_effect = lambda key: config[key] - mock_config2.side_effect = lambda key: config[key] - ctxt = ceph_hooks.get_ceph_context() - expected = {'auth_supported': False, - 'ceph_cluster_network': '', - 'ceph_public_network': '', - 'cluster_addr': '10.1.0.1', - 'dio': 'true', - 'fsid': '1234', - 'loglevel': 1, - 'mon_hosts': '10.0.0.1 10.0.0.2', - 'old_auth': False, - 'osd': {'osd max write size': 1024}, - 'osd_journal_size': 1024, - 'public_addr': '10.0.0.1', - 'short_object_len': True, - 'use_syslog': 'true', - 'bluestore': False, - 'bluestore_experimental': False} - self.assertEqual(ctxt, expected) - - @patch.object(ceph_hooks, 'get_public_addr', lambda *args: "10.0.0.1") - @patch.object(ceph_hooks, 'get_cluster_addr', lambda *args: "10.1.0.1") - @patch.object(ceph_hooks, 'cmp_pkgrevno', lambda *args: 1) - @patch.object(ceph_hooks, 'get_mon_hosts', lambda *args: ['10.0.0.1', - '10.0.0.2']) - @patch.object(ceph_hooks, 'get_networks', lambda *args: "") - @patch.object(ceph, 'config') - @patch.object(ceph_hooks, 'config') - def test_get_ceph_context_w_config_flags_invalid(self, mock_config, - mock_config2): - config = copy.deepcopy(CHARM_CONFIG) - config['config-flags'] = ('{"osd": {"osd max write size": 1024},' - '"foo": "bar"}') - mock_config.side_effect = lambda key: config[key] - mock_config2.side_effect = lambda key: config[key] - ctxt = ceph_hooks.get_ceph_context() - expected = {'auth_supported': False, - 'ceph_cluster_network': '', - 'ceph_public_network': '', - 'cluster_addr': '10.1.0.1', - 'dio': 'true', - 'fsid': '1234', - 'loglevel': 1, - 'mon_hosts': '10.0.0.1 10.0.0.2', - 'old_auth': False, - 'osd': {'osd max write size': 1024}, - 'osd_journal_size': 1024, - 'public_addr': '10.0.0.1', - 'short_object_len': True, - 'use_syslog': 'true', - 'bluestore': False, - 'bluestore_experimental': False} - self.assertEqual(ctxt, expected) - - def test_nrpe_dependency_installed(self): - with patch.multiple(ceph_hooks, - apt_install=DEFAULT, - rsync=DEFAULT, - log=DEFAULT, - write_file=DEFAULT, - nrpe=DEFAULT) as mocks: - ceph_hooks.update_nrpe_config() - mocks["apt_install"].assert_called_once_with( - ["python-dbus", "lockfile-progs"]) - - def test_upgrade_charm_with_nrpe_relation_installs_dependencies(self): - with patch.multiple( - ceph_hooks, - apt_install=DEFAULT, - rsync=DEFAULT, - log=DEFAULT, - write_file=DEFAULT, - nrpe=DEFAULT, - emit_cephconf=DEFAULT, - mon_relation_joined=DEFAULT, - is_relation_made=DEFAULT) as mocks, patch( - "charmhelpers.contrib.hardening.harden.config"): - mocks["is_relation_made"].return_value = True - ceph_hooks.upgrade_charm() - mocks["apt_install"].assert_called_with( - ["python-dbus", "lockfile-progs"]) - - -class StopHookTestCase(unittest.TestCase): - - @patch.object(ceph_hooks, 'ceph_conf_path') - @patch.object(ceph_hooks, 'socket') - @patch.object(ceph_hooks, 'subprocess') - @patch.object(ceph_hooks, 'service_pause') - @patch.object(ceph_hooks, 'cmp_pkgrevno') - @patch.object(ceph_hooks, 'remove_alternative') - def _test_stop(self, - remove_alternative, - cmp_pkgrevno, - service_pause, - subprocess, - socket, - ceph_conf_path, - ceph_mgr=False): - if ceph_mgr: - cmp_pkgrevno.return_value = 1 - else: - cmp_pkgrevno.return_value = -1 - socket.gethostname.return_value = 'myself' - ceph_conf_path.return_value = '/var/lib/charm/me/ceph.conf' - ceph_hooks.stop() - subprocess.check_call.assert_called_with( - ['ceph', 'mon', 'remove', 'myself'] - ) - if ceph_mgr: - service_pause.assert_has_calls([ - call('ceph-mon'), - call('ceph-mgr@myself') - ]) - else: - service_pause.assert_called_once_with('ceph-mon') - - remove_alternative.assert_called_with('ceph.conf', - '/var/lib/charm/me/ceph.conf') - - def test_stop_jewel(self): - self._test_stop() - - def test_stop_luminous(self): - self._test_stop(ceph_mgr=True) diff --git a/unit_tests/test_ceph_networking.py b/unit_tests/test_ceph_networking.py deleted file mode 100644 index 168e82f..0000000 --- a/unit_tests/test_ceph_networking.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import test_utils -import charmhelpers.core.hookenv as hookenv -import utils as ceph_utils - -TO_PATCH_SPACES = [ - 'network_get_primary_address', - 'log', - 'get_host_ip', - 'config', - 'get_network_addrs', - 'cached', -] - - -class CephNetworkSpaceTestCase(test_utils.CharmTestCase): - def setUp(self): - super(CephNetworkSpaceTestCase, self).setUp(ceph_utils, - TO_PATCH_SPACES) - self.config.side_effect = self.test_config.get - - def tearDown(self): - # Reset @cached cache - hookenv.cache = {} - - def test_no_network_space_support(self): - self.get_host_ip.return_value = '192.168.2.1' - self.network_get_primary_address.side_effect = NotImplementedError - self.assertEqual(ceph_utils.get_cluster_addr(), - '192.168.2.1') - self.assertEqual(ceph_utils.get_public_addr(), - '192.168.2.1') - - def test_public_network_space(self): - self.network_get_primary_address.return_value = '10.20.40.2' - self.assertEqual(ceph_utils.get_public_addr(), - '10.20.40.2') - self.network_get_primary_address.assert_called_with('public') - self.config.assert_called_with('ceph-public-network') - - def test_cluster_network_space(self): - self.network_get_primary_address.return_value = '10.20.50.2' - self.assertEqual(ceph_utils.get_cluster_addr(), - '10.20.50.2') - self.network_get_primary_address.assert_called_with('cluster') - self.config.assert_called_with('ceph-cluster-network') - - def test_config_options_in_use(self): - self.get_network_addrs.return_value = ['192.122.20.2'] - self.test_config.set('ceph-cluster-network', '192.122.20.0/24') - self.assertEqual(ceph_utils.get_cluster_addr(), - '192.122.20.2') diff --git a/unit_tests/test_ceph_ops.py b/unit_tests/test_ceph_ops.py deleted file mode 100644 index 5f17e03..0000000 --- a/unit_tests/test_ceph_ops.py +++ /dev/null @@ -1,241 +0,0 @@ -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import unittest - -from mock import ( - call, - patch, -) - -from ceph import broker - - -class TestCephOps(unittest.TestCase): - - @patch.object(broker, 'create_erasure_profile') - @patch.object(broker, 'log', lambda *args, **kwargs: None) - def test_create_erasure_profile(self, mock_create_erasure): - req = json.dumps({'api-version': 1, - 'ops': [{ - 'op': 'create-erasure-profile', - 'name': 'foo', - 'erasure-type': 'jerasure', - 'failure-domain': 'rack', - 'k': 3, - 'm': 2, - }]}) - rc = broker.process_requests(req) - mock_create_erasure.assert_called_with(service='admin', - profile_name='foo', - coding_chunks=2, - data_chunks=3, - locality=None, - failure_domain='rack', - erasure_plugin_name='jerasure') - self.assertEqual(json.loads(rc), {'exit-code': 0}) - - @patch.object(broker, 'pool_exists') - @patch.object(broker, 'ReplicatedPool') - @patch.object(broker, 'log', lambda *args, **kwargs: None) - def test_process_requests_create_replicated_pool(self, - mock_replicated_pool, - mock_pool_exists): - mock_pool_exists.return_value = False - reqs = json.dumps({'api-version': 1, - 'ops': [{ - 'op': 'create-pool', - 'pool-type': 'replicated', - 'name': 'foo', - 'replicas': 3 - }]}) - rc = broker.process_requests(reqs) - mock_pool_exists.assert_called_with(service='admin', name='foo') - calls = [call(name=u'foo', service='admin', replicas=3)] - mock_replicated_pool.assert_has_calls(calls) - self.assertEqual(json.loads(rc), {'exit-code': 0}) - - @patch.object(broker, 'pool_exists') - @patch.object(broker, 'ReplicatedPool') - @patch.object(broker, 'log', lambda *args, **kwargs: None) - def test_process_requests_replicated_pool_weight(self, - mock_replicated_pool, - mock_pool_exists): - mock_pool_exists.return_value = False - reqs = json.dumps({'api-version': 1, - 'ops': [{ - 'op': 'create-pool', - 'pool-type': 'replicated', - 'name': 'foo', - 'weight': 40.0, - 'replicas': 3 - }]}) - rc = broker.process_requests(reqs) - mock_pool_exists.assert_called_with(service='admin', name='foo') - calls = [call(name=u'foo', service='admin', replicas=3, - percent_data=40.0)] - mock_replicated_pool.assert_has_calls(calls) - self.assertEqual(json.loads(rc), {'exit-code': 0}) - - @patch.object(broker, 'delete_pool') - @patch.object(broker, 'log', lambda *args, **kwargs: None) - def test_process_requests_delete_pool(self, - mock_delete_pool): - reqs = json.dumps({'api-version': 1, - 'ops': [{ - 'op': 'delete-pool', - 'name': 'foo', - }]}) - mock_delete_pool.return_value = {'exit-code': 0} - rc = broker.process_requests(reqs) - mock_delete_pool.assert_called_with(service='admin', name='foo') - self.assertEqual(json.loads(rc), {'exit-code': 0}) - - @patch.object(broker, 'pool_exists') - @patch.object(broker.ErasurePool, 'create') - @patch.object(broker, 'erasure_profile_exists') - @patch.object(broker, 'log', lambda *args, **kwargs: None) - def test_process_requests_create_erasure_pool(self, mock_profile_exists, - mock_erasure_pool, - mock_pool_exists): - mock_pool_exists.return_value = False - reqs = json.dumps({'api-version': 1, - 'ops': [{ - 'op': 'create-pool', - 'pool-type': 'erasure', - 'name': 'foo', - 'erasure-profile': 'default' - }]}) - rc = broker.process_requests(reqs) - mock_profile_exists.assert_called_with(service='admin', name='default') - mock_pool_exists.assert_called_with(service='admin', name='foo') - mock_erasure_pool.assert_called_with() - self.assertEqual(json.loads(rc), {'exit-code': 0}) - - @patch.object(broker, 'pool_exists') - @patch.object(broker.Pool, 'add_cache_tier') - @patch.object(broker, 'log', lambda *args, **kwargs: None) - def test_process_requests_create_cache_tier(self, mock_pool, - mock_pool_exists): - mock_pool_exists.return_value = True - reqs = json.dumps({'api-version': 1, - 'ops': [{ - 'op': 'create-cache-tier', - 'cold-pool': 'foo', - 'hot-pool': 'foo-ssd', - 'mode': 'writeback', - 'erasure-profile': 'default' - }]}) - rc = broker.process_requests(reqs) - mock_pool_exists.assert_any_call(service='admin', name='foo') - mock_pool_exists.assert_any_call(service='admin', name='foo-ssd') - - mock_pool.assert_called_with(cache_pool='foo-ssd', mode='writeback') - self.assertEqual(json.loads(rc), {'exit-code': 0}) - - @patch.object(broker, 'pool_exists') - @patch.object(broker.Pool, 'remove_cache_tier') - @patch.object(broker, 'log', lambda *args, **kwargs: None) - def test_process_requests_remove_cache_tier(self, mock_pool, - mock_pool_exists): - mock_pool_exists.return_value = True - reqs = json.dumps({'api-version': 1, - 'ops': [{ - 'op': 'remove-cache-tier', - 'hot-pool': 'foo-ssd', - }]}) - rc = broker.process_requests(reqs) - mock_pool_exists.assert_any_call(service='admin', name='foo-ssd') - - mock_pool.assert_called_with(cache_pool='foo-ssd') - self.assertEqual(json.loads(rc), {'exit-code': 0}) - - @patch.object(broker, 'snapshot_pool') - @patch.object(broker, 'log', lambda *args, **kwargs: None) - def test_snapshot_pool(self, mock_snapshot_pool): - reqs = json.dumps({'api-version': 1, - 'ops': [{ - 'op': 'snapshot-pool', - 'name': 'foo', - 'snapshot-name': 'foo-snap1', - }]}) - mock_snapshot_pool.return_value = {'exit-code': 0} - rc = broker.process_requests(reqs) - mock_snapshot_pool.assert_called_with(service='admin', - pool_name='foo', - snapshot_name='foo-snap1') - self.assertEqual(json.loads(rc), {'exit-code': 0}) - - @patch.object(broker, 'rename_pool') - @patch.object(broker, 'log', lambda *args, **kwargs: None) - def test_rename_pool(self, mock_rename_pool): - reqs = json.dumps({'api-version': 1, - 'ops': [{ - 'op': 'rename-pool', - 'name': 'foo', - 'new-name': 'foo2', - }]}) - mock_rename_pool.return_value = {'exit-code': 0} - rc = broker.process_requests(reqs) - mock_rename_pool.assert_called_with(service='admin', - old_name='foo', - new_name='foo2') - self.assertEqual(json.loads(rc), {'exit-code': 0}) - - @patch.object(broker, 'remove_pool_snapshot') - @patch.object(broker, 'log', lambda *args, **kwargs: None) - def test_remove_pool_snapshot(self, mock_snapshot_pool): - reqs = json.dumps({'api-version': 1, - 'ops': [{ - 'op': 'remove-pool-snapshot', - 'name': 'foo', - 'snapshot-name': 'foo-snap1', - }]}) - mock_snapshot_pool.return_value = {'exit-code': 0} - rc = broker.process_requests(reqs) - mock_snapshot_pool.assert_called_with(service='admin', - pool_name='foo', - snapshot_name='foo-snap1') - self.assertEqual(json.loads(rc), {'exit-code': 0}) - - @patch.object(broker, 'pool_set') - @patch.object(broker, 'log', lambda *args, **kwargs: None) - def test_set_pool_value(self, mock_set_pool): - reqs = json.dumps({'api-version': 1, - 'ops': [{ - 'op': 'set-pool-value', - 'name': 'foo', - 'key': 'size', - 'value': 3, - }]}) - mock_set_pool.return_value = {'exit-code': 0} - rc = broker.process_requests(reqs) - mock_set_pool.assert_called_with(service='admin', - pool_name='foo', - key='size', - value=3) - self.assertEqual(json.loads(rc), {'exit-code': 0}) - - @patch.object(broker, 'log', lambda *args, **kwargs: None) - def test_set_invalid_pool_value(self): - reqs = json.dumps({'api-version': 1, - 'ops': [{ - 'op': 'set-pool-value', - 'name': 'foo', - 'key': 'size', - 'value': 'abc', - }]}) - rc = broker.process_requests(reqs) - self.assertEqual(json.loads(rc)['exit-code'], 1) diff --git a/unit_tests/test_config.py b/unit_tests/test_config.py deleted file mode 100644 index 66f2836..0000000 --- a/unit_tests/test_config.py +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import shutil -import sys -import tempfile - -import test_utils - -from mock import patch, MagicMock - -# python-apt is not installed as part of test-requirements but is imported by -# some charmhelpers modules so create a fake import. -mock_apt = MagicMock() -sys.modules['apt'] = mock_apt -mock_apt.apt_pkg = MagicMock() - -with patch('charmhelpers.contrib.hardening.harden.harden') as mock_dec: - mock_dec.side_effect = (lambda *dargs, **dkwargs: lambda f: - lambda *args, **kwargs: f(*args, **kwargs)) - import ceph_hooks as hooks - -TO_PATCH = [ - 'config', -] - - -class GetDevicesTestCase(test_utils.CharmTestCase): - - def setUp(self): - super(GetDevicesTestCase, self).setUp(hooks, TO_PATCH) - self.config.side_effect = self.test_config.get - self.tmp_dir = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, self.tmp_dir) - - def test_get_devices_empty(self): - """ - If osd-devices is set to an empty string, get_devices() returns - an empty list. - """ - self.test_config.set("osd-devices", "") - self.assertEqual([], hooks.get_devices()) - - def test_get_devices_non_existing_files(self): - """ - If osd-devices points to a file that doesn't exist, it's still - returned by get_devices(). - """ - non_existing = os.path.join(self.tmp_dir, "no-such-file") - self.test_config.set("osd-devices", non_existing) - self.assertEqual([non_existing], hooks.get_devices()) - - def test_get_devices_multiple(self): - """ - Multiple devices can be specified in osd-devices by separating - them with spaces. - """ - device1 = os.path.join(self.tmp_dir, "device1") - device2 = os.path.join(self.tmp_dir, "device2") - self.test_config.set("osd-devices", "{} {}".format(device1, device2)) - self.assertEqual([device1, device2], hooks.get_devices()) - - def test_get_devices_extra_spaces(self): - """ - Multiple spaces do not result in additional devices. - """ - device1 = os.path.join(self.tmp_dir, "device1") - device2 = os.path.join(self.tmp_dir, "device2") - self.test_config.set("osd-devices", "{} {}".format(device1, device2)) - self.assertEqual([device1, device2], hooks.get_devices()) - - def test_get_devices_non_absolute_path(self): - """ - Charm does not allow relative paths as this may result in a path - on the root device/within the charm directory. - """ - device1 = os.path.join(self.tmp_dir, "device1") - device2 = "foo" - self.test_config.set("osd-devices", "{} {}".format(device1, device2)) - self.assertEqual([device1], hooks.get_devices()) - - def test_get_devices_symlink(self): - """ - If a symlink is specified in osd-devices, get_devices() resolves - it and returns the link target. - """ - device = os.path.join(self.tmp_dir, "device") - link = os.path.join(self.tmp_dir, "link") - os.symlink(device, link) - self.test_config.set("osd-devices", link) - self.assertEqual([device], hooks.get_devices()) diff --git a/unit_tests/test_status.py b/unit_tests/test_status.py deleted file mode 100644 index 2e694b8..0000000 --- a/unit_tests/test_status.py +++ /dev/null @@ -1,117 +0,0 @@ -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import mock -import test_utils - -with mock.patch('charmhelpers.contrib.hardening.harden.harden') as mock_dec: - mock_dec.side_effect = (lambda *dargs, **dkwargs: lambda f: - lambda *args, **kwargs: f(*args, **kwargs)) - import ceph_hooks as hooks - -TO_PATCH = [ - 'status_set', - 'config', - 'ceph', - 'relation_ids', - 'relation_get', - 'related_units', - 'local_unit', - 'application_version_set', - 'get_upstream_version', -] - -NO_PEERS = { - 'ceph-mon1': True -} - -ENOUGH_PEERS_INCOMPLETE = { - 'ceph-mon1': True, - 'ceph-mon2': False, - 'ceph-mon3': False, -} - -ENOUGH_PEERS_COMPLETE = { - 'ceph-mon1': True, - 'ceph-mon2': True, - 'ceph-mon3': True, -} - - -class ServiceStatusTestCase(test_utils.CharmTestCase): - def setUp(self): - super(ServiceStatusTestCase, self).setUp(hooks, TO_PATCH) - self.config.side_effect = self.test_config.get - self.test_config.set('monitor-count', 3) - self.local_unit.return_value = 'ceph-mon1' - self.get_upstream_version.return_value = '10.2.2' - - @mock.patch.object(hooks, 'get_peer_units') - def test_assess_status_no_peers(self, _peer_units): - _peer_units.return_value = NO_PEERS - hooks.assess_status() - self.status_set.assert_called_with('blocked', mock.ANY) - self.application_version_set.assert_called_with('10.2.2') - - @mock.patch.object(hooks, 'get_peer_units') - def test_assess_status_peers_incomplete(self, _peer_units): - _peer_units.return_value = ENOUGH_PEERS_INCOMPLETE - hooks.assess_status() - self.status_set.assert_called_with('waiting', mock.ANY) - self.application_version_set.assert_called_with('10.2.2') - - @mock.patch.object(hooks, 'get_peer_units') - def test_assess_status_peers_complete_active(self, _peer_units): - _peer_units.return_value = ENOUGH_PEERS_COMPLETE - self.ceph.is_bootstrapped.return_value = True - self.ceph.is_quorum.return_value = True - hooks.assess_status() - self.status_set.assert_called_with('active', mock.ANY) - self.application_version_set.assert_called_with('10.2.2') - - @mock.patch.object(hooks, 'get_peer_units') - def test_assess_status_peers_complete_down(self, _peer_units): - _peer_units.return_value = ENOUGH_PEERS_COMPLETE - self.ceph.is_bootstrapped.return_value = False - self.ceph.is_quorum.return_value = False - hooks.assess_status() - self.status_set.assert_called_with('blocked', mock.ANY) - self.application_version_set.assert_called_with('10.2.2') - - def test_get_peer_units_no_peers(self): - self.relation_ids.return_value = ['mon:1'] - self.related_units.return_value = [] - self.assertEqual({'ceph-mon1': True}, - hooks.get_peer_units()) - - def test_get_peer_units_peers_incomplete(self): - self.relation_ids.return_value = ['mon:1'] - self.related_units.return_value = ['ceph-mon2', - 'ceph-mon3'] - self.relation_get.return_value = None - self.assertEqual({'ceph-mon1': True, - 'ceph-mon2': False, - 'ceph-mon3': False}, - hooks.get_peer_units()) - - def test_get_peer_units_peers_complete(self): - self.relation_ids.return_value = ['mon:1'] - self.related_units.return_value = ['ceph-mon2', - 'ceph-mon3'] - self.relation_get.side_effect = ['ceph-mon2', - 'ceph-mon3'] - self.assertEqual({'ceph-mon1': True, - 'ceph-mon2': True, - 'ceph-mon3': True}, - hooks.get_peer_units()) diff --git a/unit_tests/test_upgrade.py b/unit_tests/test_upgrade.py deleted file mode 100644 index ab1c8b4..0000000 --- a/unit_tests/test_upgrade.py +++ /dev/null @@ -1,88 +0,0 @@ -import unittest - -__author__ = 'Chris Holcombe ' - -from mock import patch, MagicMock, call - -from ceph_hooks import check_for_upgrade - - -def config_side_effect(*args): - if args[0] == 'source': - return 'cloud:trusty-kilo' - elif args[0] == 'key': - return 'key' - elif args[0] == 'release-version': - return 'cloud:trusty-kilo' - - -class UpgradeRollingTestCase(unittest.TestCase): - - @patch('ceph_hooks.ceph.resolve_ceph_version') - @patch('ceph_hooks.ceph.is_bootstrapped') - @patch('ceph_hooks.log') - @patch('ceph_hooks.ceph.roll_monitor_cluster') - @patch('ceph_hooks.ceph.wait_for_all_monitors_to_upgrade') - @patch('ceph_hooks.hookenv') - @patch('ceph_hooks.ceph.roll_osd_cluster') - def test_check_for_upgrade(self, - roll_osd_cluster, - hookenv, - wait_for_mons, - roll_monitor_cluster, - log, - is_bootstrapped, - version): - version.side_effect = ['firefly', 'hammer'] - is_bootstrapped.return_value = True - previous_mock = MagicMock().return_value - previous_mock.previous.return_value = "cloud:trusty-juno" - hookenv.config.side_effect = [previous_mock, - config_side_effect('source')] - check_for_upgrade() - - wait_for_mons.assert_called_with( - new_version='hammer', - upgrade_key='admin' - ) - roll_osd_cluster.assert_called_with( - new_version='hammer', - upgrade_key='admin' - ) - - roll_monitor_cluster.assert_called_with( - new_version='hammer', - upgrade_key='admin' - ) - log.assert_has_calls( - [ - call('old_version: firefly'), - call('new_version: hammer'), - call('firefly to hammer is a valid ' - 'upgrade path. Proceeding.') - ] - ) - - @patch('ceph_hooks.ceph.is_bootstrapped') - @patch('ceph_hooks.log') - @patch('ceph_hooks.ceph.roll_monitor_cluster') - @patch('ceph_hooks.ceph.wait_for_all_monitors_to_upgrade') - @patch('ceph_hooks.hookenv') - @patch('ceph_hooks.ceph.roll_osd_cluster') - def test_check_for_upgrade_not_bootstrapped(self, - roll_osd_cluster, - hookenv, - wait_for_mons, - roll_monitor_cluster, - log, - is_bootstrapped): - is_bootstrapped.return_value = False - previous_mock = MagicMock().return_value - previous_mock.previous.return_value = "cloud:trusty-juno" - hookenv.config.side_effect = [previous_mock, - config_side_effect('source')] - check_for_upgrade() - - roll_osd_cluster.assert_not_called() - - roll_monitor_cluster.assert_not_called() diff --git a/unit_tests/test_utils.py b/unit_tests/test_utils.py deleted file mode 100644 index 97d3ee8..0000000 --- a/unit_tests/test_utils.py +++ /dev/null @@ -1,135 +0,0 @@ -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import unittest -import os -import yaml - -from contextlib import contextmanager -from mock import patch, MagicMock - - -def load_config(): - ''' - Walk backwords from __file__ looking for config.yaml, load and return the - 'options' section' - ''' - config = None - f = __file__ - while config is None: - d = os.path.dirname(f) - if os.path.isfile(os.path.join(d, 'config.yaml')): - config = os.path.join(d, 'config.yaml') - break - f = d - - if not config: - logging.error('Could not find config.yaml in any parent directory ' - 'of %s. ' % f) - raise Exception - - return yaml.safe_load(open(config).read())['options'] - - -def get_default_config(): - ''' - Load default charm config from config.yaml return as a dict. - If no default is set in config.yaml, its value is None. - ''' - default_config = {} - config = load_config() - for k, v in config.iteritems(): - if 'default' in v: - default_config[k] = v['default'] - else: - default_config[k] = None - return default_config - - -class CharmTestCase(unittest.TestCase): - - def setUp(self, obj, patches): - super(CharmTestCase, self).setUp() - self.patches = patches - self.obj = obj - self.test_config = TestConfig() - self.test_relation = TestRelation() - self.patch_all() - - def patch(self, method): - _m = patch.object(self.obj, method) - mock = _m.start() - self.addCleanup(_m.stop) - return mock - - def patch_all(self): - for method in self.patches: - setattr(self, method, self.patch(method)) - - -class TestConfig(object): - - def __init__(self): - self.config = get_default_config() - - def get(self, attr=None): - if not attr: - return self.get_all() - try: - return self.config[attr] - except KeyError: - return None - - def get_all(self): - return self.config - - def set(self, attr, value): - if attr not in self.config: - raise KeyError - self.config[attr] = value - - -class TestRelation(object): - - def __init__(self, relation_data={}): - self.relation_data = relation_data - - def set(self, relation_data): - self.relation_data = relation_data - - def get(self, attr=None, unit=None, rid=None): - if attr is None: - return self.relation_data - elif attr in self.relation_data: - return self.relation_data[attr] - return None - - -@contextmanager -def patch_open(): - '''Patch open() to allow mocking both open() itself and the file that is - yielded. - - Yields the mock for "open" and "file", respectively.''' - mock_open = MagicMock(spec=open) - mock_file = MagicMock(spec=file) - - @contextmanager - def stub_open(*args, **kwargs): - mock_open(*args, **kwargs) - yield mock_file - - with patch('__builtin__.open', stub_open): - yield mock_open, mock_file