From 88640bf510050e5853d8379e4277ae7468b3f606 Mon Sep 17 00:00:00 2001 From: Steve Wilkerson Date: Mon, 26 Jun 2017 15:27:58 -0500 Subject: [PATCH] Add helm-toolkit back to openstack-helm-addons This adds helm-toolkit to the openstack-helm-addons repository. As we look to establish common philosophies and approaches for developing charts, we should continue to use helm-toolkit as a common library for template functions Change-Id: I5ed84afb28bd394d4c307a9fc12377f194cb76c0 --- Makefile | 2 +- helm-toolkit/.gitignore | 3 + helm-toolkit/.helmignore | 27 ++++ helm-toolkit/Chart.yaml | 18 +++ helm-toolkit/Makefile | 21 +++ helm-toolkit/requirements.yaml | 15 ++ helm-toolkit/secrets/.gitkeep | 0 .../_authenticated_endpoint_uri_lookup.tpl | 44 ++++++ .../endpoints/_hostname_endpoint_lookup.tpl | 32 ++++ .../_hostname_endpoint_uri_lookup.tpl | 39 +++++ .../_keystone_endpoint_name_lookup.tpl | 26 ++++ .../_keystone_endpoint_uri_lookup.tpl | 37 +++++ .../templates/scripts/_db-init.py.tpl | 147 ++++++++++++++++++ .../templates/scripts/_ks-domain-user.sh.tpl | 71 +++++++++ .../templates/scripts/_ks-endpoints.sh.tpl | 79 ++++++++++ .../templates/scripts/_ks-service.sh.tpl | 51 ++++++ .../templates/scripts/_ks-user.sh.tpl | 83 ++++++++++ .../templates/scripts/_rally_test.sh.tpl | 28 ++++ .../snippets/_k8s_init_dep_check.tpl | 46 ++++++ .../templates/snippets/_ks_env_openrc.tpl | 54 +++++++ .../snippets/_ks_env_user_create_openrc.tpl | 47 ++++++ .../utils/_comma_joined_hostname_list.tpl | 19 +++ helm-toolkit/templates/utils/_hash.tpl | 21 +++ .../templates/utils/_joinListWithComma.tpl | 17 ++ helm-toolkit/templates/utils/_template.tpl | 21 +++ helm-toolkit/templates/utils/_to_ini.tpl | 28 ++++ .../utils/secret-generator/README.rst | 78 ++++++++++ .../utils/secret-generator/ceph-key.py | 30 ++++ .../secret-generator/generate_secrets.sh | 96 ++++++++++++ .../templates/ceph/admin.keyring.tmpl | 20 +++ .../templates/ceph/bootstrap.keyring.tmpl | 17 ++ .../templates/ceph/ceph.conf.tmpl | 85 ++++++++++ .../templates/ceph/mon.keyring.tmpl | 17 ++ helm-toolkit/utils/test/ceph-rbd-test.yaml | 40 +++++ helm-toolkit/values.yaml | 26 ++++ tools/gate/funcs/helm.sh | 1 + 36 files changed, 1385 insertions(+), 1 deletion(-) create mode 100644 helm-toolkit/.gitignore create mode 100644 helm-toolkit/.helmignore create mode 100644 helm-toolkit/Chart.yaml create mode 100644 helm-toolkit/Makefile create mode 100644 helm-toolkit/requirements.yaml create mode 100644 helm-toolkit/secrets/.gitkeep create mode 100644 helm-toolkit/templates/endpoints/_authenticated_endpoint_uri_lookup.tpl create mode 100644 helm-toolkit/templates/endpoints/_hostname_endpoint_lookup.tpl create mode 100644 helm-toolkit/templates/endpoints/_hostname_endpoint_uri_lookup.tpl create mode 100644 helm-toolkit/templates/endpoints/_keystone_endpoint_name_lookup.tpl create mode 100644 helm-toolkit/templates/endpoints/_keystone_endpoint_uri_lookup.tpl create mode 100644 helm-toolkit/templates/scripts/_db-init.py.tpl create mode 100644 helm-toolkit/templates/scripts/_ks-domain-user.sh.tpl create mode 100755 helm-toolkit/templates/scripts/_ks-endpoints.sh.tpl create mode 100644 helm-toolkit/templates/scripts/_ks-service.sh.tpl create mode 100644 helm-toolkit/templates/scripts/_ks-user.sh.tpl create mode 100644 helm-toolkit/templates/scripts/_rally_test.sh.tpl create mode 100644 helm-toolkit/templates/snippets/_k8s_init_dep_check.tpl create mode 100644 helm-toolkit/templates/snippets/_ks_env_openrc.tpl create mode 100644 helm-toolkit/templates/snippets/_ks_env_user_create_openrc.tpl create mode 100644 helm-toolkit/templates/utils/_comma_joined_hostname_list.tpl create mode 100644 helm-toolkit/templates/utils/_hash.tpl create mode 100644 helm-toolkit/templates/utils/_joinListWithComma.tpl create mode 100644 helm-toolkit/templates/utils/_template.tpl create mode 100644 helm-toolkit/templates/utils/_to_ini.tpl create mode 100644 helm-toolkit/utils/secret-generator/README.rst create mode 100644 helm-toolkit/utils/secret-generator/ceph-key.py create mode 100755 helm-toolkit/utils/secret-generator/generate_secrets.sh create mode 100644 helm-toolkit/utils/secret-generator/templates/ceph/admin.keyring.tmpl create mode 100644 helm-toolkit/utils/secret-generator/templates/ceph/bootstrap.keyring.tmpl create mode 100644 helm-toolkit/utils/secret-generator/templates/ceph/ceph.conf.tmpl create mode 100644 helm-toolkit/utils/secret-generator/templates/ceph/mon.keyring.tmpl create mode 100644 helm-toolkit/utils/test/ceph-rbd-test.yaml create mode 100644 helm-toolkit/values.yaml diff --git a/Makefile b/Makefile index 7c316149..b6702c7a 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ HELM = helm TASK = build -CHARTS = postgresql +CHARTS = helm-toolkit postgresql all: $(CHARTS) diff --git a/helm-toolkit/.gitignore b/helm-toolkit/.gitignore new file mode 100644 index 00000000..e1bd7e85 --- /dev/null +++ b/helm-toolkit/.gitignore @@ -0,0 +1,3 @@ +secrets/* +!secrets/.gitkeep +templates/_secrets.tpl diff --git a/helm-toolkit/.helmignore b/helm-toolkit/.helmignore new file mode 100644 index 00000000..e8ef5ffa --- /dev/null +++ b/helm-toolkit/.helmignore @@ -0,0 +1,27 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj + +bin/ +etc/ +patches/ +*.py +Makefile diff --git a/helm-toolkit/Chart.yaml b/helm-toolkit/Chart.yaml new file mode 100644 index 00000000..d853b797 --- /dev/null +++ b/helm-toolkit/Chart.yaml @@ -0,0 +1,18 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +description: A base chart for all openstack charts +name: helm-toolkit +version: 0.1.0 diff --git a/helm-toolkit/Makefile b/helm-toolkit/Makefile new file mode 100644 index 00000000..9662e57a --- /dev/null +++ b/helm-toolkit/Makefile @@ -0,0 +1,21 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +EXCLUDE := templates/* charts/* Chart.yaml requirement* values.yaml Makefile utils/* helm-toolkit/Chart.yaml +SECRETS := $(shell find secrets -type f $(foreach e,$(EXCLUDE), -not -path "$(e)") ) + +templates/_secrets.tpl: Makefile $(SECRETS) + echo Generating $(CURDIR)/$@ + rm -f $@ + for i in $(SECRETS); do printf '{{ define "'$$i'" }}' >> $@; cat $$i >> $@; printf "{{ end }}\n" >> $@; done diff --git a/helm-toolkit/requirements.yaml b/helm-toolkit/requirements.yaml new file mode 100644 index 00000000..7a4ed34e --- /dev/null +++ b/helm-toolkit/requirements.yaml @@ -0,0 +1,15 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dependencies: [] diff --git a/helm-toolkit/secrets/.gitkeep b/helm-toolkit/secrets/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/helm-toolkit/templates/endpoints/_authenticated_endpoint_uri_lookup.tpl b/helm-toolkit/templates/endpoints/_authenticated_endpoint_uri_lookup.tpl new file mode 100644 index 00000000..c2928682 --- /dev/null +++ b/helm-toolkit/templates/endpoints/_authenticated_endpoint_uri_lookup.tpl @@ -0,0 +1,44 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This function helps resolve database style endpoints: +# +# Presuming that .Values contains an endpoint: definition for 'neutron-db' with the +# appropriate attributes, a call such as: +# { tuple "neutron-db" "internal" "userClass" "portName" . | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" } +# where portName is optional if a default port has been defined in .Values +# returns: mysql+pymysql://username:password@internal_host:3306/dbname + +{{- define "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" -}} +{{- $type := index . 0 -}} +{{- $endpoint := index . 1 -}} +{{- $userclass := index . 2 -}} +{{- $port := index . 3 -}} +{{- $context := index . 4 -}} +{{- $endpointMap := index $context.Values.endpoints $type }} +{{- $userMap := index $endpointMap.auth $userclass }} +{{- $fqdn := $context.Release.Namespace -}} +{{- if $context.Values.endpoints.fqdn -}} +{{- $fqdn := $context.Values.endpoints.fqdn -}} +{{- end -}} +{{- with $endpointMap -}} +{{- $endpointScheme := .scheme }} +{{- $endpointUser := index $userMap "username" }} +{{- $endpointPass := index $userMap "password" }} +{{- $endpointHost := index .hosts $endpoint | default .hosts.default}} +{{- $endpointPort := index .port $port | default .port.default }} +{{- $endpointPath := .path | default "" }} +{{- printf "%s://%s:%s@%s.%s:%1.f%s" $endpointScheme $endpointUser $endpointPass $endpointHost $fqdn $endpointPort $endpointPath -}} +{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/endpoints/_hostname_endpoint_lookup.tpl b/helm-toolkit/templates/endpoints/_hostname_endpoint_lookup.tpl new file mode 100644 index 00000000..de813ab4 --- /dev/null +++ b/helm-toolkit/templates/endpoints/_hostname_endpoint_lookup.tpl @@ -0,0 +1,32 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This function returns hostnames from endpoint definitions for use cases +# where the uri style return is not appropriate, and only the short hostname or +# kubernetes servicename is used or relevant in the template: +# { tuple "memcache" "internal" . | include "helm-toolkit.endpoints.hostname_endpoint_lookup" } +# returns: the short internal hostname, which will also match the service name + + +{{- define "helm-toolkit.endpoints.hostname_endpoint_lookup" -}} +{{- $type := index . 0 -}} +{{- $endpoint := index . 1 -}} +{{- $context := index . 2 -}} +{{- $endpointMap := index $context.Values.endpoints $type }} +{{- with $endpointMap -}} +{{- $endpointScheme := .scheme }} +{{- $endpointHost := index .hosts $endpoint | default .hosts.default}} +{{- printf "%s" $endpointHost -}} +{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/endpoints/_hostname_endpoint_uri_lookup.tpl b/helm-toolkit/templates/endpoints/_hostname_endpoint_uri_lookup.tpl new file mode 100644 index 00000000..1917a371 --- /dev/null +++ b/helm-toolkit/templates/endpoints/_hostname_endpoint_uri_lookup.tpl @@ -0,0 +1,39 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This function returns hostnames from endpoint definitions for use cases +# where the uri style return is not appropriate, and only the hostname +# portion is used or relevant in the template: +# { tuple "memcache" "internal" "portName" . | include "helm-toolkit.endpoints.hostname_endpoint_uri_lookup" } +# returns: internal_host:port +# +# Output that requires the port aspect striped could simply split the output based on ':' + +{{- define "helm-toolkit.endpoints.hostname_endpoint_uri_lookup" -}} +{{- $type := index . 0 -}} +{{- $endpoint := index . 1 -}} +{{- $port := index . 2 -}} +{{- $context := index . 3 -}} +{{- $endpointMap := index $context.Values.endpoints $type }} +{{- $fqdn := $context.Release.Namespace -}} +{{- if $context.Values.endpoints.fqdn -}} +{{- $fqdn := $context.Values.endpoints.fqdn -}} +{{- end -}} +{{- with $endpointMap -}} +{{- $endpointScheme := .scheme }} +{{- $endpointHost := index .hosts $endpoint | default .hosts.default}} +{{- $endpointPort := index .port $port | default .port.default }} +{{- printf "%s.%s:%1.f" $endpointHost $fqdn $endpointPort -}} +{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/endpoints/_keystone_endpoint_name_lookup.tpl b/helm-toolkit/templates/endpoints/_keystone_endpoint_name_lookup.tpl new file mode 100644 index 00000000..7717ec84 --- /dev/null +++ b/helm-toolkit/templates/endpoints/_keystone_endpoint_name_lookup.tpl @@ -0,0 +1,26 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This function is used in endpoint management templates +# it returns the service type for an openstack service eg: +# { tuple orchestration . | include "ks_endpoint_type" } +# will return "heat" + +{{- define "helm-toolkit.endpoints.keystone_endpoint_name_lookup" -}} +{{- $type := index . 0 -}} +{{- $context := index . 1 -}} +{{- $endpointMap := index $context.Values.endpoints $type }} +{{- $endpointName := index $endpointMap "name" }} +{{- $endpointName | quote -}} +{{- end -}} diff --git a/helm-toolkit/templates/endpoints/_keystone_endpoint_uri_lookup.tpl b/helm-toolkit/templates/endpoints/_keystone_endpoint_uri_lookup.tpl new file mode 100644 index 00000000..40189402 --- /dev/null +++ b/helm-toolkit/templates/endpoints/_keystone_endpoint_uri_lookup.tpl @@ -0,0 +1,37 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This function returns the endpoint uri for a service, it takes an tuple +# input in the form: service-type, endpoint-class, port-name. eg: +# { tuple "orchestration" "public" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" } +# will return the appropriate URI. + +{{- define "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" -}} +{{- $type := index . 0 -}} +{{- $endpoint := index . 1 -}} +{{- $port := index . 2 -}} +{{- $context := index . 3 -}} +{{- $endpointMap := index $context.Values.endpoints $type }} +{{- $fqdn := $context.Release.Namespace -}} +{{- if $context.Values.endpoints.fqdn -}} +{{- $fqdn := $context.Values.endpoints.fqdn -}} +{{- end -}} +{{- with $endpointMap -}} +{{- $endpointScheme := .scheme }} +{{- $endpointHost := index .hosts $endpoint | default .hosts.default}} +{{- $endpointPort := index .port $port }} +{{- $endpointPath := .path | default "" }} +{{- printf "%s://%s.%s:%1.f%s" $endpointScheme $endpointHost $fqdn $endpointPort $endpointPath -}} +{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/scripts/_db-init.py.tpl b/helm-toolkit/templates/scripts/_db-init.py.tpl new file mode 100644 index 00000000..cccb62de --- /dev/null +++ b/helm-toolkit/templates/scripts/_db-init.py.tpl @@ -0,0 +1,147 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- define "helm-toolkit.scripts.db_init" }} +#!/usr/bin/env python + +# Creates db and user for an OpenStack Service: +# Set ROOT_DB_CONNECTION and DB_CONNECTION environment variables to contain +# SQLAlchemy strings for the root connection to the database and the one you +# wish the service to use. Alternatively, you can use an ini formatted config +# at the location specified by OPENSTACK_CONFIG_FILE, and extract the string +# from the key OPENSTACK_CONFIG_DB_KEY, in the section specified by +# OPENSTACK_CONFIG_DB_SECTION. + +import os +import sys +import ConfigParser +import logging +from sqlalchemy import create_engine + +# Create logger, console handler and formatter +logger = logging.getLogger('OpenStack-Helm DB Init') +logger.setLevel(logging.DEBUG) +ch = logging.StreamHandler() +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + +# Set the formatter and add the handler +ch.setFormatter(formatter) +logger.addHandler(ch) + + +# Get the connection string for the service db root user +if "ROOT_DB_CONNECTION" in os.environ: + db_connection = os.environ['ROOT_DB_CONNECTION'] + logger.info('Got DB root connection') +else: + logger.critical('environment variable ROOT_DB_CONNECTION not set') + sys.exit(1) + +# Get the connection string for the service db +if "OPENSTACK_CONFIG_FILE" in os.environ: + try: + os_conf = os.environ['OPENSTACK_CONFIG_FILE'] + if "OPENSTACK_CONFIG_DB_SECTION" in os.environ: + os_conf_section = os.environ['OPENSTACK_CONFIG_DB_SECTION'] + else: + logger.critical('environment variable OPENSTACK_CONFIG_DB_SECTION not set') + sys.exit(1) + if "OPENSTACK_CONFIG_DB_KEY" in os.environ: + os_conf_key = os.environ['OPENSTACK_CONFIG_DB_KEY'] + else: + logger.critical('environment variable OPENSTACK_CONFIG_DB_KEY not set') + sys.exit(1) + config = ConfigParser.RawConfigParser() + logger.info("Using {0} as db config source".format(os_conf)) + config.read(os_conf) + logger.info("Trying to load db config from {0}:{1}".format( + os_conf_section, os_conf_key)) + user_db_conn = config.get(os_conf_section, os_conf_key) + logger.info("Got config from {0}".format(os_conf)) + except: + logger.critical("Tried to load config from {0} but failed.".format(os_conf)) + sys.exit(1) +elif "DB_CONNECTION" in os.environ: + user_db_conn = os.environ['DB_CONNECTION'] + logger.info('Got config from DB_CONNECTION env var') +else: + logger.critical('Could not get db config, either from config file or env var') + sys.exit(1) + +# Root DB engine +try: + root_engine_full = create_engine(db_connection) + root_user = root_engine_full.url.username + root_password = root_engine_full.url.password + drivername = root_engine_full.url.drivername + host = root_engine_full.url.host + port = root_engine_full.url.port + root_engine_url = ''.join([drivername, '://', root_user, ':', root_password, '@', host, ':', str (port)]) + root_engine = create_engine(root_engine_url) + connection = root_engine.connect() + connection.close() + logger.info("Tested connection to DB @ {0}:{1} as {2}".format( + host, port, root_user)) +except: + logger.critical('Could not connect to database as root user') + raise + sys.exit(1) + +# User DB engine +try: + user_engine = create_engine(user_db_conn) + # Get our user data out of the user_engine + database = user_engine.url.database + user = user_engine.url.username + password = user_engine.url.password + logger.info('Got user db config') +except: + logger.critical('Could not get user database config') + raise + sys.exit(1) + +# Create DB +try: + root_engine.execute("CREATE DATABASE IF NOT EXISTS {0}".format(database)) + logger.info("Created database {0}".format(database)) +except: + logger.critical("Could not create database {0}".format(database)) + raise + sys.exit(1) + +# Create DB User +try: + root_engine.execute( + "GRANT ALL ON `{0}`.* TO \'{1}\'@\'%%\' IDENTIFIED BY \'{2}\'".format( + database, user, password)) + logger.info("Created user {0} for {1}".format(user, database)) +except: + logger.critical("Could not create user {0} for {1}".format(user, database)) + raise + sys.exit(1) + +# Test connection +try: + connection = user_engine.connect() + connection.close() + logger.info("Tested connection to DB @ {0}:{1}/{2} as {3}".format( + host, port, database, user)) +except: + logger.critical('Could not connect to database as user') + raise + sys.exit(1) + +logger.info('Finished DB Management') +{{- end }} diff --git a/helm-toolkit/templates/scripts/_ks-domain-user.sh.tpl b/helm-toolkit/templates/scripts/_ks-domain-user.sh.tpl new file mode 100644 index 00000000..0680f91c --- /dev/null +++ b/helm-toolkit/templates/scripts/_ks-domain-user.sh.tpl @@ -0,0 +1,71 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- define "helm-toolkit.scripts.keystone_domain_user" }} +#!/bin/bash + +# Copyright 2017 Pete Birley +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# Manage domain +SERVICE_OS_DOMAIN_ID=$(openstack domain create --or-show --enable -f value -c id \ + --description="Service Domain for ${SERVICE_OS_REGION_NAME}/${SERVICE_OS_DOMAIN_NAME}" \ + "${SERVICE_OS_DOMAIN_NAME}") + +# Display domain +openstack domain show "${SERVICE_OS_DOMAIN_ID}" + +# Manage user +SERVICE_OS_USERID=$(openstack user create --or-show --enable -f value -c id \ + --domain="${SERVICE_OS_DOMAIN_ID}" \ + --description "Service User for ${SERVICE_OS_REGION_NAME}/${SERVICE_OS_DOMAIN_NAME}" \ + --password="${SERVICE_OS_PASSWORD}" \ + "${SERVICE_OS_USERNAME}") + +# Display user +openstack user show "${SERVICE_OS_USERID}" + +# Manage role +SERVICE_OS_ROLE_ID=$(openstack role show -f value -c id \ + --domain="${SERVICE_OS_DOMAIN_ID}" \ + "${SERVICE_OS_ROLE}" || openstack role create -f value -c id \ + --domain="${SERVICE_OS_DOMAIN_ID}" \ + "${SERVICE_OS_ROLE}" ) + +# Manage user role assignment +openstack role add \ + --domain="${SERVICE_OS_DOMAIN_ID}" \ + --user="${SERVICE_OS_USERID}" \ + --user-domain="${SERVICE_OS_DOMAIN_ID}" \ + "${SERVICE_OS_ROLE_ID}" + +# Display user role assignment +openstack role assignment list \ + --role="${SERVICE_OS_ROLE_ID}" \ + --user-domain="${SERVICE_OS_DOMAIN_ID}" \ + --user="${SERVICE_OS_USERID}" +{{- end }} diff --git a/helm-toolkit/templates/scripts/_ks-endpoints.sh.tpl b/helm-toolkit/templates/scripts/_ks-endpoints.sh.tpl new file mode 100755 index 00000000..0c19a85e --- /dev/null +++ b/helm-toolkit/templates/scripts/_ks-endpoints.sh.tpl @@ -0,0 +1,79 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- define "helm-toolkit.scripts.keystone_endpoints" }} +#!/bin/bash + +# Copyright 2017 Pete Birley +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# Get Service ID +OS_SERVICE_ID=$( openstack service list -f csv --quote none | \ + grep ",${OS_SERVICE_NAME},${OS_SERVICE_TYPE}$" | \ + sed -e "s/,${OS_SERVICE_NAME},${OS_SERVICE_TYPE}//g" ) + +# Get Endpoint ID if it exists +OS_ENDPOINT_ID=$( openstack endpoint list -f csv --quote none | \ + grep "^[a-z0-9]*,${OS_REGION_NAME},${OS_SERVICE_NAME},${OS_SERVICE_TYPE},True,${OS_SVC_ENDPOINT}," | \ + awk -F ',' '{ print $1 }' ) + +# Making sure only a single endpoint exists for a service within a region +if [ "$(echo $OS_ENDPOINT_ID | wc -w)" -gt "1" ]; then + echo "More than one endpoint found, cleaning up" + for ENDPOINT_ID in $OS_ENDPOINT_ID; do + openstack endpoint delete ${ENDPOINT_ID} + done + unset OS_ENDPOINT_ID +fi + +# Determine if Endpoint needs updated +if [[ ${OS_ENDPOINT_ID} ]]; then + OS_ENDPOINT_URL_CURRENT=$(openstack endpoint show ${OS_ENDPOINT_ID} --f value -c url) + if [ "${OS_ENDPOINT_URL_CURRENT}" == "${OS_SERVICE_ENDPOINT}" ]; then + echo "Endpoints Match: no action required" + OS_ENDPOINT_UPDATE="False" + else + echo "Endpoints Dont Match: removing existing entries" + openstack endpoint delete ${OS_ENDPOINT_ID} + OS_ENDPOINT_UPDATE="True" + fi +else + OS_ENDPOINT_UPDATE="True" +fi + +# Update Endpoint if required +if [[ "${OS_ENDPOINT_UPDATE}" == "True" ]]; then + OS_ENDPOINT_ID=$( openstack endpoint create -f value -c id \ + --region="${OS_REGION_NAME}" \ + "${OS_SERVICE_ID}" \ + ${OS_SVC_ENDPOINT} \ + "${OS_SERVICE_ENDPOINT}" ) +fi + +# Display the Endpoint +openstack endpoint show ${OS_ENDPOINT_ID} +{{- end }} diff --git a/helm-toolkit/templates/scripts/_ks-service.sh.tpl b/helm-toolkit/templates/scripts/_ks-service.sh.tpl new file mode 100644 index 00000000..ea812e88 --- /dev/null +++ b/helm-toolkit/templates/scripts/_ks-service.sh.tpl @@ -0,0 +1,51 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- define "helm-toolkit.scripts.keystone_service" }} +#!/bin/bash + +# Copyright 2017 Pete Birley +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# Service boilerplate description +OS_SERVICE_DESC="${OS_REGION_NAME}: ${OS_SERVICE_NAME} (${OS_SERVICE_TYPE}) service" + +# Get Service ID if it exists +unset OS_SERVICE_ID +OS_SERVICE_ID=$( openstack service list -f csv --quote none | \ + grep ",${OS_SERVICE_NAME},${OS_SERVICE_TYPE}$" | \ + sed -e "s/,${OS_SERVICE_NAME},${OS_SERVICE_TYPE}//g" ) + +# If a Service ID was not found, then create the service +if [[ -z ${OS_SERVICE_ID} ]]; then + OS_SERVICE_ID=$(openstack service create -f value -c id \ + --name="${OS_SERVICE_NAME}" \ + --description "${OS_SERVICE_DESC}" \ + --enable \ + "${OS_SERVICE_TYPE}") +fi +{{- end }} diff --git a/helm-toolkit/templates/scripts/_ks-user.sh.tpl b/helm-toolkit/templates/scripts/_ks-user.sh.tpl new file mode 100644 index 00000000..71121a38 --- /dev/null +++ b/helm-toolkit/templates/scripts/_ks-user.sh.tpl @@ -0,0 +1,83 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- define "helm-toolkit.scripts.keystone_user" }} +#!/bin/bash + +# Copyright 2017 Pete Birley +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# Manage user project +USER_PROJECT_DESC="Service Project for ${SERVICE_OS_REGION_NAME}/${SERVICE_OS_PROJECT_DOMAIN_NAME}" +USER_PROJECT_ID=$(openstack project create --or-show --enable -f value -c id \ + --domain="${SERVICE_OS_PROJECT_DOMAIN_NAME}" \ + --description="${USER_PROJECT_DESC}" \ + "${SERVICE_OS_PROJECT_NAME}"); + +# Display project +openstack project show "${USER_PROJECT_ID}" + +# Manage user +USER_DESC="Service User for ${SERVICE_OS_REGION_NAME}/${SERVICE_OS_USER_DOMAIN_NAME}/${SERVICE_OS_SERVICE_NAME}" +USER_ID=$(openstack user create --or-show --enable -f value -c id \ + --domain="${SERVICE_OS_USER_DOMAIN_NAME}" \ + --project-domain="${SERVICE_OS_PROJECT_DOMAIN_NAME}" \ + --project="${USER_PROJECT_ID}" \ + --description="${USER_DESC}" \ + --password="${SERVICE_OS_PASSWORD}" \ + "${SERVICE_OS_USERNAME}"); + +# Display user +openstack user show "${USER_ID}" + +function ks_assign_user_role () { + # Manage user role assignment + openstack role add \ + --user="${USER_ID}" \ + --user-domain="${SERVICE_OS_USER_DOMAIN_NAME}" \ + --project-domain="${SERVICE_OS_PROJECT_DOMAIN_NAME}" \ + --project="${USER_PROJECT_ID}" \ + "${USER_ROLE_ID}" + + # Display user role assignment + openstack role assignment list \ + --role="${USER_ROLE_ID}" \ + --user-domain="${SERVICE_OS_USER_DOMAIN_NAME}" \ + --user="${USER_ID}" +} + +# Manage user service role +export USER_ROLE_ID=$(openstack role create --or-show -f value -c id \ + "${SERVICE_OS_ROLE}"); +ks_assign_user_role + +# Manage user member role +: ${MEMBER_OS_ROLE:="_member_"} +export USER_ROLE_ID=$(openstack role create --or-show -f value -c id \ + "${MEMBER_OS_ROLE}"); +ks_assign_user_role +{{- end }} diff --git a/helm-toolkit/templates/scripts/_rally_test.sh.tpl b/helm-toolkit/templates/scripts/_rally_test.sh.tpl new file mode 100644 index 00000000..4c8cf6ab --- /dev/null +++ b/helm-toolkit/templates/scripts/_rally_test.sh.tpl @@ -0,0 +1,28 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- define "helm-toolkit.scripts.rally_test" }} +#!/bin/bash +set -ex + +: ${RALLY_ENV_NAME:="openstack-helm"} +rally-manage db create +rally deployment create --fromenv --name ${RALLY_ENV_NAME} +rally deployment use ${RALLY_ENV_NAME} +rally deployment check +rally task validate /etc/rally/rally_tests.yaml +rally task start /etc/rally/rally_tests.yaml +rally deployment destroy --deployment ${RALLY_ENV_NAME} +rally task sla-check +{{- end }} diff --git a/helm-toolkit/templates/snippets/_k8s_init_dep_check.tpl b/helm-toolkit/templates/snippets/_k8s_init_dep_check.tpl new file mode 100644 index 00000000..4c0b7623 --- /dev/null +++ b/helm-toolkit/templates/snippets/_k8s_init_dep_check.tpl @@ -0,0 +1,46 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- define "helm-toolkit.snippets.kubernetes_entrypoint_init_container" -}} +{{- $envAll := index . 0 -}} +{{- $deps := index . 1 -}} +{{- $mounts := index . 2 -}} +- name: init + image: {{ $envAll.Values.images.dep_check }} + imagePullPolicy: {{ $envAll.Values.images.pull_policy }} + env: + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: INTERFACE_NAME + value: eth0 + - name: DEPENDENCY_SERVICE + value: "{{ tuple $deps.services $envAll | include "helm-toolkit.utils.comma_joined_hostname_list" }}" + - name: DEPENDENCY_JOBS + value: "{{ include "helm-toolkit.utils.joinListWithComma" $deps.jobs }}" + - name: DEPENDENCY_DAEMONSET + value: "{{ include "helm-toolkit.utils.joinListWithComma" $deps.daemonset }}" + - name: DEPENDENCY_CONTAINER + value: "{{ include "helm-toolkit.utils.joinListWithComma" $deps.container }}" + - name: COMMAND + value: "echo done" + volumeMounts: {{ $mounts | default "[]"}} +{{- end -}} diff --git a/helm-toolkit/templates/snippets/_ks_env_openrc.tpl b/helm-toolkit/templates/snippets/_ks_env_openrc.tpl new file mode 100644 index 00000000..28565013 --- /dev/null +++ b/helm-toolkit/templates/snippets/_ks_env_openrc.tpl @@ -0,0 +1,54 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- define "helm-toolkit.snippets.keystone_openrc_env_vars" }} +{{- $ksUserSecret := .ksUserSecret }} +- name: OS_IDENTITY_API_VERSION + value: "3" +- name: OS_AUTH_URL + valueFrom: + secretKeyRef: + name: {{ $ksUserSecret }} + key: OS_AUTH_URL +- name: OS_REGION_NAME + valueFrom: + secretKeyRef: + name: {{ $ksUserSecret }} + key: OS_REGION_NAME +- name: OS_PROJECT_DOMAIN_NAME + valueFrom: + secretKeyRef: + name: {{ $ksUserSecret }} + key: OS_PROJECT_DOMAIN_NAME +- name: OS_PROJECT_NAME + valueFrom: + secretKeyRef: + name: {{ $ksUserSecret }} + key: OS_PROJECT_NAME +- name: OS_USER_DOMAIN_NAME + valueFrom: + secretKeyRef: + name: {{ $ksUserSecret }} + key: OS_USER_DOMAIN_NAME +- name: OS_USERNAME + valueFrom: + secretKeyRef: + name: {{ $ksUserSecret }} + key: OS_USERNAME +- name: OS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $ksUserSecret }} + key: OS_PASSWORD +{{- end }} diff --git a/helm-toolkit/templates/snippets/_ks_env_user_create_openrc.tpl b/helm-toolkit/templates/snippets/_ks_env_user_create_openrc.tpl new file mode 100644 index 00000000..f9a73e7b --- /dev/null +++ b/helm-toolkit/templates/snippets/_ks_env_user_create_openrc.tpl @@ -0,0 +1,47 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- define "helm-toolkit.snippets.keystone_user_create_env_vars" }} +{{- $ksUserSecret := .ksUserSecret }} +- name: SERVICE_OS_REGION_NAME + valueFrom: + secretKeyRef: + name: {{ $ksUserSecret }} + key: OS_REGION_NAME +- name: SERVICE_OS_PROJECT_DOMAIN_NAME + valueFrom: + secretKeyRef: + name: {{ $ksUserSecret }} + key: OS_PROJECT_DOMAIN_NAME +- name: SERVICE_OS_PROJECT_NAME + valueFrom: + secretKeyRef: + name: {{ $ksUserSecret }} + key: OS_PROJECT_NAME +- name: SERVICE_OS_USER_DOMAIN_NAME + valueFrom: + secretKeyRef: + name: {{ $ksUserSecret }} + key: OS_USER_DOMAIN_NAME +- name: SERVICE_OS_USERNAME + valueFrom: + secretKeyRef: + name: {{ $ksUserSecret }} + key: OS_USERNAME +- name: SERVICE_OS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $ksUserSecret }} + key: OS_PASSWORD +{{- end }} diff --git a/helm-toolkit/templates/utils/_comma_joined_hostname_list.tpl b/helm-toolkit/templates/utils/_comma_joined_hostname_list.tpl new file mode 100644 index 00000000..f697d616 --- /dev/null +++ b/helm-toolkit/templates/utils/_comma_joined_hostname_list.tpl @@ -0,0 +1,19 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- define "helm-toolkit.utils.comma_joined_hostname_list" -}} +{{- $deps := index . 0 -}} +{{- $envAll := index . 1 -}} +{{- range $k, $v := $deps -}}{{- if $k -}},{{- end -}}{{ tuple $v.service $v.endpoint $envAll | include "helm-toolkit.endpoints.hostname_endpoint_lookup" }}{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/utils/_hash.tpl b/helm-toolkit/templates/utils/_hash.tpl new file mode 100644 index 00000000..110990bb --- /dev/null +++ b/helm-toolkit/templates/utils/_hash.tpl @@ -0,0 +1,21 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- define "helm-toolkit.utils.hash" -}} +{{- $name := index . 0 -}} +{{- $context := index . 1 -}} +{{- $last := base $context.Template.Name }} +{{- $wtf := $context.Template.Name | replace $last $name -}} +{{- include $wtf $context | sha256sum | quote -}} +{{- end -}} diff --git a/helm-toolkit/templates/utils/_joinListWithComma.tpl b/helm-toolkit/templates/utils/_joinListWithComma.tpl new file mode 100644 index 00000000..2a7c691c --- /dev/null +++ b/helm-toolkit/templates/utils/_joinListWithComma.tpl @@ -0,0 +1,17 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- define "helm-toolkit.utils.joinListWithComma" -}} +{{ range $k, $v := . }}{{ if $k }},{{ end }}{{ $v }}{{ end }} +{{- end -}} diff --git a/helm-toolkit/templates/utils/_template.tpl b/helm-toolkit/templates/utils/_template.tpl new file mode 100644 index 00000000..d3f6cda3 --- /dev/null +++ b/helm-toolkit/templates/utils/_template.tpl @@ -0,0 +1,21 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- define "helm-toolkit.utils.template" -}} +{{- $name := index . 0 -}} +{{- $context := index . 1 -}} +{{- $last := base $context.Template.Name }} +{{- $wtf := $context.Template.Name | replace $last $name -}} +{{ include $wtf $context }} +{{- end -}} diff --git a/helm-toolkit/templates/utils/_to_ini.tpl b/helm-toolkit/templates/utils/_to_ini.tpl new file mode 100644 index 00000000..f8f4a369 --- /dev/null +++ b/helm-toolkit/templates/utils/_to_ini.tpl @@ -0,0 +1,28 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- define "helm-toolkit.utils.to_ini" -}} +{{- range $section, $values := . -}} +{{- if kindIs "map" $values -}} +[{{ $section }}] +{{range $key, $value := $values -}} +{{- if kindIs "slice" $value -}} +{{ $key }} = {{ include "helm-toolkit.joinListWithComma" $value }} +{{else -}} +{{ $key }} = {{ $value }} +{{end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/helm-toolkit/utils/secret-generator/README.rst b/helm-toolkit/utils/secret-generator/README.rst new file mode 100644 index 00000000..ace977e6 --- /dev/null +++ b/helm-toolkit/utils/secret-generator/README.rst @@ -0,0 +1,78 @@ +Ceph Kubernetes Secret Generation +================================= + +This script will generate ceph keyrings and configs as Kubernetes +secrets. + +Sigil is required for template handling and must be installed in system +``PATH``. Instructions can be found`here +`__ + +The following functions are provided: + +Generate raw FSID (can be used for other functions) +--------------------------------------------------- + +.. code:: bash + + ./generate_secrets.sh fsid + +Generate raw ceph.conf (For verification) +----------------------------------------- + +.. code:: bash + + ./generate_secrets.sh ceph-conf-raw "overridekey=value" + +Take a look at ``ceph/ceph.conf.tmpl`` for the default values + +Generate encoded ceph.conf secret +--------------------------------- + +.. code:: bash + + ./generate_secrets.sh ceph-conf "overridekey=value" + +Generate encoded admin keyring secret +------------------------------------- + +.. code:: bash + + ./generate_secrets.sh admin-keyring + +Generate encoded mon keyring secret +----------------------------------- + +.. code:: bash + + ./generate_secrets.sh mon-keyring + +Generate a combined secret +-------------------------- + +Contains ceph.conf, admin keyring and mon keyring. Useful for generating +the ``/etc/ceph`` directory + +.. code:: bash + + ./generate_secrets.sh combined-conf + +Generate encoded boostrap keyring secret +---------------------------------------- + +.. code:: bash + + ./generate_secrets.sh bootstrap-keyring + +Kubernetes workflow +=================== + +.. code:: bash + + ./generator/generate_secrets.sh all `./generate_secrets.sh fsid` + + kubectl create secret generic ceph-conf-combined --from-file=ceph.conf --from-file=ceph.client.admin.keyring --from-file=ceph.mon.keyring --namespace=ceph + kubectl create secret generic ceph-bootstrap-rgw-keyring --from-file=ceph.keyring=ceph.rgw.keyring --namespace=ceph + kubectl create secret generic ceph-bootstrap-mds-keyring --from-file=ceph.keyring=ceph.mds.keyring --namespace=ceph + kubectl create secret generic ceph-bootstrap-osd-keyring --from-file=ceph.keyring=ceph.osd.keyring --namespace=ceph + kubectl create secret generic ceph-client-key --from-file=ceph-client-key --namespace=ceph diff --git a/helm-toolkit/utils/secret-generator/ceph-key.py b/helm-toolkit/utils/secret-generator/ceph-key.py new file mode 100644 index 00000000..761f41a8 --- /dev/null +++ b/helm-toolkit/utils/secret-generator/ceph-key.py @@ -0,0 +1,30 @@ +#!/bin/python + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import struct +import time +import base64 + +key = os.urandom(16) +header = struct.pack( + ' ../../secrets/ceph.conf + + key=$(python ceph-key.py) + keyring=$(sigil -f templates/ceph/admin.keyring.tmpl "key=${key}") + echo "${key}" > ../../secrets/ceph-client-key + echo "${keyring}" > ../../secrets/ceph.client.admin.keyring + + key=$(python ceph-key.py) + keyring=$(sigil -f templates/ceph/mon.keyring.tmpl "key=${key}") + echo "${keyring}" > ../../secrets/ceph.mon.keyring +} + +gen-bootstrap-keyring() { + service="${1:-osd}" + key=$(python ceph-key.py) + bootstrap=$(sigil -f templates/ceph/bootstrap.keyring.tmpl "key=${key}" "service=${service}") + echo "${bootstrap}" +} + +gen-all-bootstrap-keyrings() { + gen-bootstrap-keyring osd > ../../secrets/ceph.osd.keyring + gen-bootstrap-keyring mds > ../../secrets/ceph.mds.keyring + gen-bootstrap-keyring rgw > ../../secrets/ceph.rgw.keyring +} + +gen-all() { + gen-combined-conf $@ + gen-all-bootstrap-keyrings +} + + +main() { + set -eo pipefail + case "$1" in + fsid) shift; gen-fsid $@;; + ceph-conf-raw) shift; gen-ceph-conf-raw $@;; + ceph-conf) shift; gen-ceph-conf $@;; + admin-keyring) shift; gen-admin-keyring $@;; + mon-keyring) shift; gen-mon-keyring $@;; + bootstrap-keyring) shift; gen-bootstrap-keyring $@;; + combined-conf) shift; gen-combined-conf $@;; + all) shift; gen-all $@;; + esac +} + +main "$@" diff --git a/helm-toolkit/utils/secret-generator/templates/ceph/admin.keyring.tmpl b/helm-toolkit/utils/secret-generator/templates/ceph/admin.keyring.tmpl new file mode 100644 index 00000000..b0f04dba --- /dev/null +++ b/helm-toolkit/utils/secret-generator/templates/ceph/admin.keyring.tmpl @@ -0,0 +1,20 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[client.admin] + key = {{ $key }} + auid = 0 + caps mds = "allow" + caps mon = "allow *" + caps osd = "allow *" diff --git a/helm-toolkit/utils/secret-generator/templates/ceph/bootstrap.keyring.tmpl b/helm-toolkit/utils/secret-generator/templates/ceph/bootstrap.keyring.tmpl new file mode 100644 index 00000000..639a1010 --- /dev/null +++ b/helm-toolkit/utils/secret-generator/templates/ceph/bootstrap.keyring.tmpl @@ -0,0 +1,17 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[client.bootstrap-{{ $service }}] + key = {{ $key }} + caps mon = "allow profile bootstrap-{{ $service }}" diff --git a/helm-toolkit/utils/secret-generator/templates/ceph/ceph.conf.tmpl b/helm-toolkit/utils/secret-generator/templates/ceph/ceph.conf.tmpl new file mode 100644 index 00000000..b7c925bf --- /dev/null +++ b/helm-toolkit/utils/secret-generator/templates/ceph/ceph.conf.tmpl @@ -0,0 +1,85 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[global] +fsid = ${fsid:?} +cephx = ${auth_cephx:-"true"} +cephx_require_signatures = ${auth_cephx_require_signatures:-"false"} +cephx_cluster_require_signatures = ${auth_cephx_cluster_require_signatures:-"true"} +cephx_service_require_signatures = ${auth_cephx_service_require_signatures:-"false"} + +# auth +max_open_files = ${global_max_open_files:-"131072"} +osd_pool_default_pg_num = ${global_osd_pool_default_pg_num:-"128"} +osd_pool_default_pgp_num = ${global_osd_pool_default_pgp_num:-"128"} +osd_pool_default_size = ${global_osd_pool_default_size:-"3"} +osd_pool_default_min_size = ${global_osd_pool_default_min_size:-"1"} + +mon_osd_full_ratio = ${global_mon_osd_full_ratio:-".95"} +mon_osd_nearfull_ratio = ${global_mon_osd_nearfull_ratio:-".85"} + +mon_host = ${global_mon_host:-'ceph-mon'} + +[mon] +mon_osd_down_out_interval = ${mon_mon_osd_down_out_interval:-"600"} +mon_osd_min_down_reporters = ${mon_mon_osd_min_down_reporters:-"4"} +mon_clock_drift_allowed = ${mon_mon_clock_drift_allowed:-".15"} +mon_clock_drift_warn_backoff = ${mon_mon_clock_drift_warn_backoff:-"30"} +mon_osd_report_timeout = ${mon_mon_osd_report_timeout:-"300"} + + +[osd] +journal_size = ${osd_journal_size:-"100"} +cluster_network = ${osd_cluster_network:-'192.168.0.0/16'} +public_network = ${osd_public_network:-'192.168.0.0/16'} +osd_mkfs_type = ${osd_osd_mkfs_type:-"xfs"} +osd_mkfs_options_xfs = ${osd_osd_mkfs_options_xfs:-"-f -i size=2048"} +osd_mon_heartbeat_interval = ${osd_osd_mon_heartbeat_interval:-"30"} +osd_max_object_name_len = ${osd_max_object_name_len:-"256"} + +#crush +osd_pool_default_crush_rule = ${osd_pool_default_crush_rule:-"0"} +osd_crush_update_on_start = ${osd_osd_crush_update_on_start:-"true"} + +#backend +osd_objectstore = ${osd_osd_objectstore:-"filestore"} + +#performance tuning +filestore_merge_threshold = ${osd_filestore_merge_threshold:-"40"} +filestore_split_multiple = ${osd_filestore_split_multiple:-"8"} +osd_op_threads = ${osd_osd_op_threads:-"8"} +filestore_op_threads = ${osd_filestore_op_threads:-"8"} +filestore_max_sync_interval = ${osd_filestore_max_sync_interval:-"5"} +osd_max_scrubs = ${osd_osd_max_scrubs:-"1"} + + +#recovery tuning +osd_recovery_max_active = ${osd_osd_recovery_max_active:-"5"} +osd_max_backfills = ${osd_osd_max_backfills:-"2"} +osd_recovery_op_priority = ${osd_osd_recovery_op_priority:-"2"} +osd_client_op_priority = ${osd_osd_client_op_priority:-"63"} +osd_recovery_max_chunk = ${osd_osd_recovery_max_chunk:-"1048576"} +osd_recovery_threads = ${osd_osd_recovery_threads:-"1"} + +#ports +ms_bind_port_min = ${osd_ms_bind_port_min:-"6800"} +ms_bind_port_max = ${osd_ms_bind_port_max:-"7100"} + +[client] +rbd_cache_enabled = ${client_rbd_cache_enabled:-"true"} +rbd_cache_writethrough_until_flush = ${client_rbd_cache_writethrough_until_flush:-"true"} +rbd_default_features = ${client_rbd_default_features:-"1"} + +[mds] +mds_cache_size = ${mds_mds_cache_size:-"100000"} diff --git a/helm-toolkit/utils/secret-generator/templates/ceph/mon.keyring.tmpl b/helm-toolkit/utils/secret-generator/templates/ceph/mon.keyring.tmpl new file mode 100644 index 00000000..9a88a242 --- /dev/null +++ b/helm-toolkit/utils/secret-generator/templates/ceph/mon.keyring.tmpl @@ -0,0 +1,17 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[mon.] + key = {{ $key }} + caps mon = "allow *" diff --git a/helm-toolkit/utils/test/ceph-rbd-test.yaml b/helm-toolkit/utils/test/ceph-rbd-test.yaml new file mode 100644 index 00000000..32b91d2a --- /dev/null +++ b/helm-toolkit/utils/test/ceph-rbd-test.yaml @@ -0,0 +1,40 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Pod +metadata: + name: ceph-rbd-test +spec: + containers: + - name: cephrbd-rw + image: busybox + command: + - sh + - -c + - while true; do sleep 1; done + volumeMounts: + - mountPath: "/mnt/cephrbd" + name: cephrbd + volumes: + - name: cephrbd + rbd: + monitors: +#This only works if you have skyDNS resolveable from the kubernetes node. Otherwise you must manually put in one or more mon pod ips. + - ceph-mon.ceph:6789 + user: admin + image: ceph-rbd-test + pool: rbd + secretRef: + name: ceph-client-key diff --git a/helm-toolkit/values.yaml b/helm-toolkit/values.yaml new file mode 100644 index 00000000..9a2b0c22 --- /dev/null +++ b/helm-toolkit/values.yaml @@ -0,0 +1,26 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for utils. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value + +global: + region: cluster + tld: local + +endpoints: + fqdn: null + diff --git a/tools/gate/funcs/helm.sh b/tools/gate/funcs/helm.sh index f5a04857..3eb1c915 100755 --- a/tools/gate/funcs/helm.sh +++ b/tools/gate/funcs/helm.sh @@ -69,6 +69,7 @@ function helm_serve { } function helm_lint { + make build-helm-toolkit -C ${WORK_DIR} make TASK=lint -C ${WORK_DIR} }