From 539613ce4f96435fe4ad97290e4af2b35288bac7 Mon Sep 17 00:00:00 2001 From: Anastasia Kuznetsova Date: Tue, 14 Jun 2016 16:15:04 +0300 Subject: [PATCH] Add new version of tests - Created new tests structure - Refactored and partially rewrote code from ugly.py - Added tox.ini Change-Id: I154906b8fcc7870b82fd2b782d03e029d9dd1df2 --- .testr.conf | 4 + setup.cfg | 22 +++ setup.py | 21 +++ test-requirements.txt | 9 + tests/__init__.py | 0 tests/base.py | 331 ++++++++++++++++++++++++++++++++++ tests/clients.py | 84 +++++++++ tests/test_cicd_apps.py | 70 ++++++++ tools/jenkins/run.sh | 34 ---- tools/jenkins/ugly.py | 384 ---------------------------------------- tox.ini | 31 ++++ 11 files changed, 572 insertions(+), 418 deletions(-) create mode 100644 .testr.conf create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 test-requirements.txt create mode 100755 tests/__init__.py create mode 100755 tests/base.py create mode 100755 tests/clients.py create mode 100755 tests/test_cicd_apps.py delete mode 100755 tools/jenkins/run.sh delete mode 100755 tools/jenkins/ugly.py create mode 100644 tox.ini diff --git a/.testr.conf b/.testr.conf new file mode 100644 index 0000000..ece4541 --- /dev/null +++ b/.testr.conf @@ -0,0 +1,4 @@ +[DEFAULT] +test_command=OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 ${PYTHON:-python} -m subunit.run discover -t ./tests -s ${SUBUNIT_TEST_PATH:-./tests/} $LISTOPT $IDOPTION +test_id_option=--load-list $IDFILE +test_list_option=--list diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..6ef5709 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,22 @@ +[metadata] +name = refstack +summary = OpenStack interop testing +description-file = + README.rst +author = OpenStack +author-email = openstack-dev@lists.openstack.org +home-page = http://www.openstack.org/ +classifier = + Environment :: OpenStack + Intended Audience :: Developers + Intended Audience :: Information Technology + License :: OSI Approved :: Apache Software License + Operating System :: POSIX :: Linux + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3.3 + +[global] +setup-hooks = + pbr.hooks.setup_hook diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..37540d1 --- /dev/null +++ b/setup.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +# Copyright (c) 2014 Piston Cloud Computing, inc. all rights reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import setuptools + +setuptools.setup( + setup_requires=['pbr'], +pbr=True) diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..f8e6b08 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,9 @@ +testrepository +testtools +python-heatclient +python-keystoneclient +requests +# FIXME: revert to pip client, when issue with yaql will be fixed +#python-muranoclient +git+https://github.com/openstack/python-muranoclient.git@master + diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/tests/base.py b/tests/base.py new file mode 100755 index 0000000..5bd024d --- /dev/null +++ b/tests/base.py @@ -0,0 +1,331 @@ +# Copyright (c) 2016 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import os +import socket +import time +import uuid + +import requests +import testtools +import yaml +import muranoclient.common.exceptions as exceptions + +import clients + +ARTIFACTS_DIR = os.environ.get('ARTIFACTS_DIR', 'logs') + +LOG = logging.getLogger(__name__) +LOG.setLevel(logging.DEBUG) +if not os.path.exists(ARTIFACTS_DIR): + os.makedirs(ARTIFACTS_DIR) +fh = logging.FileHandler(os.path.join(ARTIFACTS_DIR, 'runner.log')) +formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s' +) +fh.setFormatter(formatter) +LOG.addHandler(fh) + +ch = logging.StreamHandler() +ch.setLevel(logging.DEBUG) +ch.setFormatter(formatter) +LOG.addHandler(ch) + +# Sometimes need to pass some boolean from bash env. Since each bash +# variable is string, we need such simply hack +_boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True, + '0': False, 'no': False, 'false': False, 'off': False} + + +def str2bool(name, default): + value = os.environ.get(name, '') + return _boolean_states.get(value.lower(), default) + + +class MuranoTestsBase(testtools.TestCase, clients.ClientsBase): + + def setUp(self): + super(MuranoTestsBase, self).setUp() + # counter, for murano deployment logger + self.latest_report = 0 + self.flavor = os.environ.get('OS_FLAVOR', 'm1.medium') + self.image = os.environ.get('OS_IMAGE') + self.keyname = os.environ.get('OS_KEYNAME', None) + self.availability_zone = os.environ.get('OS_ZONE', 'nova') + self.deploy_timeout = (60 * 60) * 2 + # Since its really useful to debug deployment after it fail...lets + # add such possibility + self.os_cleanup_before = str2bool('OS_CLEANUP_BEFORE', False) + self.os_cleanup_after = str2bool('OS_CLEANUP_AFTER', True) + + self.keystone = self.initialize_keystone_client() + self.heat = self.initialize_heat_client(self.keystone) + self.murano = self.initialize_murano_client(self.keystone) + self.headers = { + 'X-Auth-Token': self.murano.http_client.auth_token, + 'content-type': 'application/json' + } + self.envs = [] + if self.os_cleanup_before: + self.cleanup_up_tenant() + LOG.info('Running test: {0}'.format(self._testMethodName)) + + def tearDown(self): + if not self.os_cleanup_after: + for env in self.envs: + try: + self.delete_env(env) + except Exception: + self.delete_stack(env) + + super(MuranoTestsBase, self).tearDown() + + @staticmethod + def rand_name(name='murano_ci_test_'): + return name + str(time.strftime("%Y_%m_%d_%H_%M_%S")) + + @staticmethod + def generate_id(): + return uuid.uuid4() + + def _get_stack(self, environment_id): + for stack in self.heat.stacks.list(): + if environment_id in stack.description: + return stack + + def cleanup_up_tenant(self): + LOG.debug('Removing EVERYTHING in tenant: {0}'.format( + self.keystone.tenant_name)) + for env in self.murano.environments.list(): + self.delete_env(env) + for stack in self.heat.stacks.list(): + try: + self.heat.stacks.delete(stack.id) + except Exception as e: + LOG.warning("Unable delete stack:{}".format(stack)) + LOG.exception(e) + pass + return + + def delete_stack(self, environment): + stack = self._get_stack(environment.id) + if not stack: + return + else: + self.heat.stacks.delete(stack.id) + + def create_env(self): + name = self.rand_name() + environment = self.murano.environments.create({'name': name}) + self.envs.append(environment) + self.addCleanup(self.delete_env, environment) + LOG.debug('Created Environment:\n {0}'.format(environment)) + + return environment + + def delete_env(self, environment, timeout=360): + try: + self.murano.environments.delete(environment.id) + start_time = time.time() + while time.time() - start_time < timeout: + try: + self.murano.environments.get(environment.id) + time.sleep(1) + except exceptions.HTTPNotFound: + return + raise exceptions.HTTPOverLimit( + 'Environment "{0}" was not deleted in {1} seconds'.format( + environment.id, timeout) + ) + except (exceptions.HTTPForbidden, exceptions.HTTPOverLimit, + exceptions.HTTPNotFound): + try: + self.murano.environments.delete(environment.id, abandon=True) + LOG.warning( + 'Environment "{0}" from test {1} abandoned'.format( + environment.id, self._testMethodName)) + except exceptions.HTTPNotFound: + return + + start_time = time.time() + while time.time() - start_time < timeout: + try: + self.murano.environments.get(environment.id) + time.sleep(1) + except exceptions.HTTPNotFound: + return + raise Exception( + 'Environment "{0}" was not deleted in {1} seconds'.format( + environment.id, timeout) + ) + + def get_env(self, environment): + return self.murano.environments.get(environment.id) + + def deploy_env(self, environment, session): + self.murano.sessions.deploy(environment.id, session.id) + return self.wait_for_environment_deploy(environment) + + def get_deployment_report(self, environment, deployment): + history = '' + report = self.murano.deployments.reports(environment.id, deployment.id) + for status in report: + history += '\t{0} - {1}\n'.format(status.created, status.text) + return history + + def _log_report(self, environment): + deployment = self.murano.deployments.list(environment.id)[0] + details = deployment.result['result']['details'] + LOG.error('Exception found:\n {0}'.format(details)) + report = self.get_deployment_report(environment, deployment) + LOG.debug('Report:\n {0}\n'.format(report)) + + def _log_latest(self, environment): + deployment = self.murano.deployments.list(environment.id)[0] + history = self.get_deployment_report(environment, deployment) + if self.latest_report != len(history) or self.latest_report == 0: + tmp = len(history) + history = history[self.latest_report:] + LOG.debug("Last report from murano engine:\n{}".format((history))) + self.latest_report = tmp + return history + + def wait_for_environment_deploy(self, env): + start_time = time.time() + status = self.get_env(env).manager.get(env.id).status + + while status != 'ready': + status = self.get_env(env).manager.get(env.id).status + LOG.debug('Deployment status:{}...nothing new..'.format(status)) + self._log_latest(env) + + if time.time() - start_time > self.deploy_timeout: + time.sleep(60) + self.fail( + 'Environment deployment wasn\'t' + 'finished in {} seconds'.format(self.deploy_timeout) + ) + elif status == 'deploy failure': + self._log_report(env) + self.fail( + 'Environment has incorrect status "{0}"'.format(status) + ) + + time.sleep(30) + LOG.debug('Environment "{0}" is ready'.format(self.get_env(env).name)) + return self.get_env(env).manager.get(env.id) + + def create_session(self, environment): + return self.murano.sessions.configure(environment.id) + + def create_service(self, environment, session, json_data, to_json=True): + LOG.debug('Adding service:\n {0}'.format(json_data)) + service = self.murano.services.post( + environment.id, + path='/', + data=json_data, + session_id=session.id + ) + if to_json: + service = service.to_dict() + service = json.dumps(service) + LOG.debug('Create Service json: {0}'.format(yaml.load(service))) + return yaml.load(service) + else: + LOG.debug('Create Service: {0}'.format(service)) + return service + + @staticmethod + def guess_fip(env_obj_model): + + result = {} + + def _finditem(obj, result): + if 'floatingIpAddress' in obj.get('instance', []): + result[obj['?']['package']] = obj['instance'][ + 'floatingIpAddress'] + for k, v in obj.items(): + if isinstance(v, dict): + _finditem(v, result) + _finditem(env_obj_model, result) + + return result + + def check_ports_open(self, ip, ports): + for port in ports: + result = 1 + start_time = time.time() + while time.time() - start_time < 60: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + result = sock.connect_ex((str(ip), port)) + sock.close() + + if result == 0: + LOG.debug('{} port is opened on instance'.format(port)) + break + time.sleep(5) + + if result != 0: + self.fail('{} port is not opened on instance'.format(port)) + + def check_url_access(self, ip, path, port): + attempt = 0 + proto = 'http' if port not in (443, 8443) else 'https' + url = '%s://%s:%s/%s' % (proto, ip, port, path) + + while attempt < 5: + resp = requests.get(url) + if resp.status_code == 200: + LOG.debug('Service path "{}" is available'.format(url)) + return + else: + time.sleep(5) + attempt += 1 + + self.fail( + 'Service path {0} is unavailable after 5 attempts'.format(url) + ) + + def deployment_success_check(self, environment, services_map): + deployment = self.murano.deployments.list(environment.id)[-1] + + self.assertEqual( + 'success', deployment.state, + 'Deployment status is "{0}"'.format(deployment.state) + ) + + fips = self.guess_fip(environment.services[0]) + + for service in services_map: + LOG.debug( + 'Checking ports availability on "{}" app instance'.format( + service) + ) + self.check_ports_open( + fips[service], services_map[service]['ports'] + ) + if services_map[service]['url']: + LOG.debug( + 'Checking {0} app url "{1}" availability'.format( + service, services_map[service]['url'] + ) + ) + self.check_url_access( + fips[service], + services_map[service]['url'], + services_map[service]['url_port'] + ) diff --git a/tests/clients.py b/tests/clients.py new file mode 100755 index 0000000..2f88e97 --- /dev/null +++ b/tests/clients.py @@ -0,0 +1,84 @@ +# Copyright (c) 2016 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from heatclient import client as heatclient +from keystoneclient.v2_0 import client as keystoneclient +from muranoclient import client as muranoclient + + +class ClientsBase(object): + + @staticmethod + def initialize_keystone_client(): + username = os.environ.get('OS_USERNAME') + password = os.environ.get('OS_PASSWORD') + tenant_name = os.environ.get('OS_TENANT_NAME') + os_auth_uri = os.environ.get('OS_AUTH_URL') + + keystone = keystoneclient.Client( + username=username, + password=password, + tenant_name=tenant_name, + auth_url=os_auth_uri + ) + return keystone + + @classmethod + def get_endpoint(cls, service_type, endpoint_type): + ks_client = cls.initialize_keystone_client() + + return ks_client.service_catalog.url_for( + service_type=service_type, + endpoint_type=endpoint_type + ) + + @classmethod + def initialize_murano_client(cls, auth_client=None): + ks_client = (auth_client if auth_client + else cls.initialize_keystone_client()) + + murano_endpoint = cls.get_endpoint( + service_type='application-catalog', + endpoint_type='publicURL' + ) + + murano = muranoclient.Client( + '1', + endpoint=murano_endpoint, + token=ks_client.auth_token + ) + + return murano + + @classmethod + def initialize_heat_client(cls, auth_client=None): + ks_client = (auth_client if auth_client + else cls.initialize_keystone_client()) + + heat_endpoint = cls.get_endpoint( + service_type='orchestration', + endpoint_type='publicURL' + ) + + heat = heatclient.Client( + '1', + endpoint=heat_endpoint, + token=ks_client.auth_token + ) + + return heat + diff --git a/tests/test_cicd_apps.py b/tests/test_cicd_apps.py new file mode 100755 index 0000000..a368ca1 --- /dev/null +++ b/tests/test_cicd_apps.py @@ -0,0 +1,70 @@ +# Copyright (c) 2016 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base + + +class MuranoCiCdTest(base.MuranoTestsBase): + + def test_deploy_cicd(self): + environment = self.create_env() + session = self.create_session(environment) + service_json = { + '?': { + '_{id}'.format(id=self.generate_id().hex): {'name': 'CI/CD'}, + 'id': str(self.generate_id()), + 'type': + 'org.openstack.ci_cd_pipeline_murano_app.CiCdEnvironment' + }, + 'assignFloatingIp': True, + 'availabilityZone': self.availability_zone, + 'flavor': self.flavor, + 'image': self.image, + 'instance_name': environment.name, + 'keyname': self.keyname, + 'ldapEmail': 'email@example.com', + 'ldapPass': 'P@ssw0rd', + 'ldapRootEmail': 'root@example.com', + 'ldapRootPass': 'P@ssw0rd', + 'ldapRootUser': 'root', + 'ldapUser': 'user', + 'name': 'CI/CD' + } + self.create_service(environment, session, service_json) + self.deploy_env(environment, session) + + environment = self.get_env(environment) + check_services = { + 'org.openstack.ci_cd_pipeline_murano_app.Jenkins': { + 'ports': [8080, 22], + 'url': 'api/', + 'url_port': 8080 + }, + 'org.openstack.ci_cd_pipeline_murano_app.Gerrit': { + 'ports': [8081, 22], + 'url': '#/admin/projects/', + 'url_port': 8081 + }, + 'org.openstack.ci_cd_pipeline_murano_app.OpenLDAP': { + 'ports': [389, 22], + 'url': None + }, + } + + self.deployment_success_check(environment, check_services) + base.LOG.debug("Run second deployment, w\o any changes..") + session = self.create_session(environment) + self.deploy_env(environment, session) + self.deployment_success_check(environment, check_services) diff --git a/tools/jenkins/run.sh b/tools/jenkins/run.sh deleted file mode 100755 index 141c229..0000000 --- a/tools/jenkins/run.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -set -e -# Requirments: -# apt-get install python-venv python-pip build-essential libssl-dev libffi-dev -y - -# Variables: -VENV_PATH=${VENV_PATH:-.venv} -VENV_CLEAN=${VENV_CLEAN:-false} -TEST_NAME=${TEST_NAME:-none} - -function prepare_venv() { - echo 'LOG: Creating python venv for murano-client' - rm -rf "${VENV_PATH}" - mkdir -p "${VENV_PATH}" - virtualenv --system-site-packages "${VENV_PATH}" - source "${VENV_PATH}/bin/activate" - #TODO install from requirments.txt ? - pip install python-muranoclient python-heatclient - deactivate -} - -# Body -if [[ ("${VENV_CLEAN}" == true) || (! -f "${VENV_PATH}/bin/activate") ]]; then - prepare_venv -fi - -if [[ "${TEST_NAME}" != "none" ]] ; then - source "${VENV_PATH}/bin/activate" - echo "LOG: Attempt to run test=${TEST_NAME}" - ./utils/jenkins/"${TEST_NAME}" - deactivate -fi - diff --git a/tools/jenkins/ugly.py b/tools/jenkins/ugly.py deleted file mode 100755 index 53c926f..0000000 --- a/tools/jenkins/ugly.py +++ /dev/null @@ -1,384 +0,0 @@ -#!/usr/bin/env python - -# Ugly script, for quick deployment. copy-paste from -# https://github.com/vryzhenkin/pegasus - -from muranoclient import client as muranocl -import os -import logging -import urlparse -import yaml -import json -import uuid -import time -import socket -import pprint -import requests -from keystoneclient.v2_0 import client as keystoneclient -from heatclient.client import Client as HeatCl - -username = os.environ.get('OS_USERNAME') -password = os.environ.get('OS_PASSWORD') -tenant_name = os.environ.get('OS_TENANT_NAME', False) -uri = os.environ.get('OS_AUTH_URL') -murano_endpoint = os.environ.get('OS_MURANO_URL') -heat_endpoint = os.environ.get('OS_HEAT_URL') - -os_cleanup = os.environ.get('OS_CLEANUP', False) -murano_env_name = os.environ.get('ENV_NAME_PREFIX', 'murano_ci_test_') -flavor = os.environ.get('OS_FLAVOR', 'm1.medium') -image = os.environ.get('OS_IMAGE', 'Ubuntu_14.04_x64_murano-agent') -keyname = os.environ.get('OS_KEYNAME', 'test_key') -availability_zone = os.environ.get('OS_ZONE', 'nova') -# to be passed to murano-app -m_pass = 'P@ssw0rd' -ARTIFACTS_DIR = os.environ.get('ARTIFACTS_DIR', 'logs') -BUILD_TAG = os.environ.get('BUILD_TAG', None) -deploy_timeout = (60 * 60) * 2 - -formatter = logging.Formatter('%(asctime)s - %(name)s - ' - '%(levelname)s - %(message)s') -LOG = logging.getLogger(__name__) -LOG.setLevel(logging.DEBUG) -if not os.path.exists(ARTIFACTS_DIR): - os.makedirs(ARTIFACTS_DIR) -fh = logging.FileHandler(os.path.join(ARTIFACTS_DIR, 'runner.log')) -fh.setLevel(logging.DEBUG) -fh.setFormatter(formatter) -LOG.addHandler(fh) -ch = logging.StreamHandler() -ch.setLevel(logging.DEBUG) -ch.setFormatter(formatter) -LOG.addHandler(ch) -pprinter = pprint.PrettyPrinter(indent=1, width=80, depth=None) -history_log = 0 - -murano_client = None - -# service name , from murano-app map, and port, which should be checked -check_map = {'org.openstack.ci_cd_pipeline_murano_app.Jenkins': - {'ports': [8080, 22], 'url': 'api/', 'url_port': 8080}, - 'org.openstack.ci_cd_pipeline_murano_app.Gerrit': - {'ports': [8081, 22], 'url': '#/admin/projects/', - 'url_port': 8081}, - 'org.openstack.ci_cd_pipeline_murano_app.OpenLDAP': - {'ports': [389, 22], 'url': None}, - } - - -def get_murano_client(): - """ - Hook for keystone - :return: - """ - global murano_client - if murano_client is None: - keystone = get_auth() - murano_client = muranocl.Client('1', endpoint=murano_endpoint, - token=keystone.auth_token) - return murano_client - try: - keystoneclient.Client(token=murano_client.http_client.auth_token, - auth_url=uri) - except Exception as e: - keystone = get_auth() - murano_client = muranocl.Client('1', endpoint=murano_endpoint, - token=keystone.auth_token) - return murano_client - - -def get_auth(): - keystone = keystoneclient.Client(username=username, - password=password, - tenant_name=tenant_name, - auth_url=uri) - return keystone - - -def rand_name(name=murano_env_name): - return name + str(time.strftime("%Y_%m_%d_%H_%M_%S")) - - -def create_env(): - if BUILD_TAG is None: - name = rand_name(murano_env_name) - else: - name = murano_env_name + str(BUILD_TAG) - environment = get_murano_client().environments.create({'name': name}) - LOG.debug( - 'Created Environment:\n{0}'.format(pprinter.pformat(environment))) - return environment.id - - -def get_env(env_id=None): - if env_id: - return get_murano_client().environments.get(env_id) - else: - fail('Wrong environment id!') - - -def add_service(env_id, data, session): - """ - This function adding a specific service to environment - Returns a specific class - :param env_id: - :param data: - :param session: - :return: - """ - LOG.debug('Added service:\n {0}'.format(data)) - return get_murano_client().services.post(env_id, - path='/', - data=data, - session_id=session.id) - - -def create_service(env_id, session, json_data, to_json=True): - """ - This function adding a specific service to environment - Returns a JSON object with a service - :param env_id: - :param session: - :param json_data: - :return: - """ - - service = add_service(env_id, json_data, session) - if to_json: - service = service.to_dict() - service = json.dumps(service) - LOG.debug('Create Service json:{0}'.format(pprinter.pformat(service))) - return yaml.load(service) - else: - LOG.debug('Create Service: {0}'.format(service)) - return service - - -def fail(msg=None): - """Fail immediately, with the given message.""" - raise Exception(msg) - - -def get_last_deployment(env_id): - deployments = get_murano_client().deployments.list(env_id) - return deployments[0] - - -def get_deployment_report(env_id, deployment, as_list=False): - report = get_murano_client().deployments.reports(env_id, deployment.id) - if as_list: - history = [] - for status in report: - history.append('{0} - {1}'.format(status.created, status.text)) - else: - history = '' - for status in report: - history += '\t{0} - {1}\n'.format(status.created, status.text) - return history - - -def _log_report(env_id): - deployment = get_last_deployment(env_id) - try: - details = deployment.result['result']['details'] - except KeyError: - LOG.error('Deployment has no result details!') - pass - LOG.error('Exception found:\n {0}'.format(details)) - report = get_deployment_report(env_id, deployment) - LOG.debug('Report:\n {0}\n'.format(report)) - - -def _log_quick(env_id): - global history_log - deployment = get_last_deployment(env_id) - history = get_deployment_report(env_id, deployment, as_list=True) - if history_log != len(history) or history_log == 0: - tmp = len(history) - history = history[history_log:] - LOG.debug("Last report:\n{}".format(pprinter.pformat(history))) - history_log = tmp - return history - - -def wait_for_environment_deploy(env_id): - start_time = time.time() - status = get_env(env_id).manager.get(env_id).status - while status != 'ready': - status = get_env(env_id).manager.get(env_id).status - LOG.debug('Deployment status:{}...nothing new..'.format(status)) - _log_quick(env_id) - if time.time() - start_time > deploy_timeout: - time.sleep(60) - _log_report(env_id) - fail( - 'Environment deployment is not finished in {}seconds'.format( - deploy_timeout)) - elif status == 'deploy failure': - _log_report(env_id) - fail('Environment has incorrect status {0}'.format(status)) - time.sleep(30) - LOG.debug('Environment {0} is ready'.format(get_env(env_id).name)) - return get_env(env_id).manager.get(env_id) - - -def deploy_environment(env_id, session): - get_murano_client().sessions.deploy(env_id, session.id) - return wait_for_environment_deploy(env_id) - - -def divine_fip(obj): - """ - will return dict like app_type : fip - - :param obj: - :param result: - :return: - """ - result = {} - - def _finditem(obj, result): - if 'floatingIpAddress' in obj.get('instance', []): - result[obj['?']['package']] = obj['instance'][ - 'floatingIpAddress'] - for k, v in obj.items(): - if isinstance(v, dict): - _finditem(v, result) - _finditem(obj, result) - return result - - -def check_port_access(ip, port): - # FIXME - result = 1 - start_time = time.time() - while time.time() - start_time < 60: - - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - result = sock.connect_ex((str(ip), port)) - sock.close() - if result == 0: - LOG.debug('%s port is opened on instance' % port) - break - else: - fail('%s port is not opened on instance' % port) - time.sleep(5) - if result != 0: - fail('%s port is not opened on instance' % port) - - -def check_path(path, ip, port=80): - attempts = 5 - proto = 'http' - if port in (443, 8443): - proto = 'https' - url = '%s://%s:%s/%s' % (proto, ip, port, path) - for i in range(attempts): - resp = requests.get(url) - i += 1 - if resp.status_code != 200 and i >= attempts: - fail('Service path failed: %s Code:%s' % (url, resp.status_code)) - elif resp.status_code == 200: - LOG.debug('Service path fine: %s' % url) - return - else: - time.sleep(5) - - -def deployment_success_check(env_id, ip, inst_name='jenkins', ports=[]): - deployment = get_murano_client().deployments.list(env_id)[-1] - LOG.debug('Deployment status is {0}'.format(deployment.state)) - if str(deployment.state) != 'success': - fail('Wrong deploymnet state = {}'.format(deployment.state)) - for port in ports: - LOG.debug("Looking into: {} {}:{} ".format(inst_name, ip, port)) - check_port_access(ip, port) - - -def cleaup_up_tenant(): - LOG.warning('Removing everything from tenant{}'.format(tenant_name)) - murano = get_murano_client() - for env in murano.environments.list(): - try: - murano.environments.delete(env.id) - except Exception as e: - LOG.warning("Unable delete env:{}".format(env.id)) - LOG.exception(e) - try: - LOG.warning("Trying abandon env:{}".format(env.id)) - murano.environments.delete(env.id, abandon=True) - except Exception as e: - LOG.warning("Unable abandon env:{}".format(env.id)) - LOG.exception(e) - pass - pass - - tenant_id = get_auth().get_project_id(tenant_name) - heat_url = urlparse.urljoin(heat_endpoint, tenant_id) - heat = HeatCl('1', endpoint=heat_url, token=get_auth().auth_token) - for stack in heat.stacks.list(): - try: - heat.stacks.delete(stack.id) - except Exception as e: - LOG.warning("Unable delete stack:{}".format(stack)) - LOG.exception(e) - pass - return - -if __name__ == '__main__': - - if os_cleanup and tenant_name.lower() == 'admin': - fail( - "Never use this ugly test with 'admin' tenant! it can " - "destroy to much!") - - if os_cleanup: - LOG.warning('Removing all stuff') - cleaup_up_tenant() - - # instant test - # murano = get_murano_client() - # environment_id = murano.environments.get('deca41a0e8504eef864') - - env_id = create_env() - session = get_murano_client().sessions.configure(env_id) - - post_body = { - '?': {'_{id}'.format(id=uuid.uuid4().hex): {'name': 'CI/CD'}, - 'id': str(uuid.uuid4()), - 'type': 'org.openstack.ci_cd_pipeline_murano_app.CiCdEnvironment'}, - 'assignFloatingIp': True, - 'availabilityZone': availability_zone, - 'flavor': flavor, - 'image': image, - 'instance_name': get_env(env_id).name, - 'keyname': keyname, - 'ldapEmail': 'email@example.com', - 'ldapPass': m_pass, - 'ldapRootEmail': 'root@example.com', - 'ldapRootPass': m_pass, - 'ldapRootUser': 'root', - 'ldapUser': 'user', - 'name': 'CI/CD' - } - - try: - create_service(env_id, session, post_body) - LOG.debug("Attempt to deploy env..") - deploy_environment(env_id, session) - fip_map = divine_fip(get_env(env_id).services[0]) - for app in check_map: - deployment_success_check(env_id, fip_map[app], app, - check_map[app]['ports']) - if check_map[app]['url']: - LOG.debug('Checking service {}'.format(app)) - check_path(check_map[app]['url'], fip_map[app], - port=check_map[app]['url_port']) - LOG.debug("Deployment finished successfully") - - except Exception as exc: - LOG.exception('Deployment error %s' % exc) - raise - finally: - deployment = get_last_deployment(env_id) - diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..983fa1e --- /dev/null +++ b/tox.ini @@ -0,0 +1,31 @@ +[tox] +minversion = 1.6 +skipsdist = True +envlist = py27 +skip_missing_interpreters = True + +[testenv] +setenv = VIRTUAL_ENV={envdir} + LANG=en_US.UTF-8 + LANGUAGE=en_US:en + LC_ALL=C +passenv = OS_* MURANO* *ENDPOINT* +deps= + pytz + extras + python-subunit + debtcollector + -r{toxinidir}/test-requirements.txt +distribute = false + +[testenv:venv] +commands = {posargs:} + +[testenv:deploy_cicd_apps] +commands = python setup.py testr --testr-args='{posargs}' + +[testenv:hacking] +deps= + ipdb + -r{toxinidir}/test-requirements.txt +commands = python -m unittest tests.test_cicd_apps.MuranoCiCdTest.test_deploy_cicd