From 65b07097b221d2c39091daac339296e277dddd66 Mon Sep 17 00:00:00 2001 From: Vladimir Kozhukalov Date: Mon, 26 Oct 2015 14:38:18 +0300 Subject: [PATCH] Deprecate shotgun directory Change-Id: Ib1260df78a45e9cc8bf6bdfd9627508ce50ae812 Related-Bug: #1506894 --- MAINTAINERS | 9 - build_docs.sh | 1 - docs/develop/env.rst | 1 - docs/develop/nailgun/development/env.rst | 2 - run_tests.sh | 36 ---- shotgun/.coveragerc | 3 - shotgun/.testr.conf | 4 - shotgun/bin/example.json | 40 ---- shotgun/bin/example.py | 34 --- shotgun/requirements.txt | 1 - shotgun/run_tests.sh | 20 -- shotgun/setup.py | 36 ---- shotgun/shotgun/__init__.py | 13 -- shotgun/shotgun/cli.py | 71 ------- shotgun/shotgun/config.py | 69 ------ shotgun/shotgun/driver.py | 257 ----------------------- shotgun/shotgun/logger.py | 37 ---- shotgun/shotgun/manager.py | 43 ---- shotgun/shotgun/settings.py | 20 -- shotgun/shotgun/test/__init__.py | 13 -- shotgun/shotgun/test/base.py | 19 -- shotgun/shotgun/test/test_config.py | 58 ----- shotgun/shotgun/test/test_driver.py | 219 ------------------- shotgun/shotgun/test/test_manager.py | 43 ---- shotgun/shotgun/test/test_utils.py | 112 ---------- shotgun/shotgun/utils.py | 158 -------------- shotgun/specs/shotgun.spec | 40 ---- shotgun/test-requirements.txt | 4 - shotgun/tox.ini | 43 ---- specs/fuel-nailgun.spec | 27 --- 30 files changed, 1433 deletions(-) delete mode 100644 shotgun/.coveragerc delete mode 100644 shotgun/.testr.conf delete mode 100644 shotgun/bin/example.json delete mode 100644 shotgun/bin/example.py delete mode 100644 shotgun/requirements.txt delete mode 100755 shotgun/run_tests.sh delete mode 100644 shotgun/setup.py delete mode 100644 shotgun/shotgun/__init__.py delete mode 100644 shotgun/shotgun/cli.py delete mode 100644 shotgun/shotgun/config.py delete mode 100644 shotgun/shotgun/driver.py delete mode 100644 shotgun/shotgun/logger.py delete mode 100644 shotgun/shotgun/manager.py delete mode 100644 shotgun/shotgun/settings.py delete mode 100644 shotgun/shotgun/test/__init__.py delete mode 100644 shotgun/shotgun/test/base.py delete mode 100644 shotgun/shotgun/test/test_config.py delete mode 100644 shotgun/shotgun/test/test_driver.py delete mode 100644 shotgun/shotgun/test/test_manager.py delete mode 100644 shotgun/shotgun/test/test_utils.py delete mode 100644 shotgun/shotgun/utils.py delete mode 100644 shotgun/specs/shotgun.spec delete mode 100644 shotgun/test-requirements.txt delete mode 100644 shotgun/tox.ini diff --git a/MAINTAINERS b/MAINTAINERS index 3afcf904c0..c76163ed84 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -49,15 +49,6 @@ maintainers: email: loles@mirantis.com IRC: salmon_ -- shotgun/: - - name: Maciej Kwiek - email: mkwiek@mirantis.com - IRC: mkwiek - - - name: Vladimir Kozhukalov - email: vkozhukalov@mirantis.com - IRC: kozhukalov - - nailgun/: - name: Aleksandr Kislitskii email: akislitsky@mirantis.com diff --git a/build_docs.sh b/build_docs.sh index 24decff7fc..b40a296ffb 100755 --- a/build_docs.sh +++ b/build_docs.sh @@ -88,7 +88,6 @@ prepare_venv() { virtualenv "${VENV}" # you can use any name instead of 'fuel' . "${VENV}/bin/activate" # command selects the particular environment # install dependencies - pip install ./shotgun # this fuel project is listed in setup.py requirements pip install -r 'nailgun/test-requirements.txt' } diff --git a/docs/develop/env.rst b/docs/develop/env.rst index d17e22258a..227549c94b 100644 --- a/docs/develop/env.rst +++ b/docs/develop/env.rst @@ -273,7 +273,6 @@ Now you can create the virtual environment and activate it. And then install the dependencies. :: - pip install ./shotgun pip install -r nailgun/test-requirements.txt Now you can look at the list of available formats and generate diff --git a/docs/develop/nailgun/development/env.rst b/docs/develop/nailgun/development/env.rst index 60103f3042..5d307b3cb0 100644 --- a/docs/develop/nailgun/development/env.rst +++ b/docs/develop/nailgun/development/env.rst @@ -78,7 +78,6 @@ Preparing Development Environment sudo apt-get install --yes git git clone https://github.com/openstack/fuel-web.git cd fuel-web - pip install ./shotgun # this fuel project is listed in setup.py requirements pip install --allow-all-external -r nailgun/test-requirements.txt #. Create required folder for log files:: @@ -302,4 +301,3 @@ you could only run dropdb with *./run_tests.sh* script. Now you need to run dropdb for each slave node: the *py.test --cleandb * command is introduced for this purpose. - diff --git a/run_tests.sh b/run_tests.sh index 4fac1bf2ed..019cc5e765 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -26,8 +26,6 @@ function usage { echo " -x, --performance Run NAILGUN performance tests" echo " -p, --flake8 Run FLAKE8 and HACKING compliance check" echo " -P, --no-flake8 Don't run static code checks" - echo " -s, --shotgun Run SHOTGUN tests" - echo " -S, --no-shotgun Don't run SHOTGUN tests" echo " -t, --tests Run a given test files" echo " -u, --upgrade Run tests for UPGRADE system" echo " -U, --no-upgrade Don't run tests for UPGRADE system" @@ -59,8 +57,6 @@ function process_options { -x|--performance) performance_tests=1;; -u|--upgrade) upgrade_system=1;; -U|--no-upgrade) no_upgrade_system=1;; - -s|--shotgun) shotgun_tests=1;; - -S|--no-shotgun) no_shotgun_tests=1;; -p|--flake8) flake8_checks=1;; -P|--no-flake8) no_flake8_checks=1;; -w|--webui) ui_lint_checks=1; ui_unit_tests=1; ui_func_tests=1;; @@ -100,7 +96,6 @@ testropts="--with-timer --timer-warning=10 --timer-ok=2 --timer-top-n=10" # nosetest xunit options NAILGUN_XUNIT=${NAILGUN_XUNIT:-"$ROOT/nailgun.xml"} FUELUPGRADE_XUNIT=${FUELUPGRADE_XUNIT:-"$ROOT/fuelupgrade.xml"} -SHOTGUN_XUNIT=${SHOTGUN_XUNIT:-"$ROOT/shotgun.xml"} EXTENSIONS_XUNIT=${EXTENSIONS_XUNIT:-"$ROOT/extensions.xml"} NAILGUN_PORT=${NAILGUN_PORT:-5544} TEST_NAILGUN_DB=${TEST_NAILGUN_DB:-nailgun} @@ -118,8 +113,6 @@ no_nailgun_tests=0 performance_tests=0 upgrade_system=0 no_upgrade_system=0 -shotgun_tests=0 -no_shotgun_tests=0 flake8_checks=0 no_flake8_checks=0 ui_lint_checks=0 @@ -162,7 +155,6 @@ function run_tests { $ui_unit_tests -eq 0 && \ $ui_func_tests -eq 0 && \ $upgrade_system -eq 0 && \ - $shotgun_tests -eq 0 && \ $extensions_tests -eq 0 && \ $flake8_checks -eq 0 ]]; then @@ -171,7 +163,6 @@ function run_tests { if [ $no_ui_unit_tests -ne 1 ]; then ui_unit_tests=1; fi if [ $no_ui_func_tests -ne 1 ]; then ui_func_tests=1; fi if [ $no_upgrade_system -ne 1 ]; then upgrade_system=1; fi - if [ $no_shotgun_tests -ne 1 ]; then shotgun_tests=1; fi if [ $no_flake8_checks -ne 1 ]; then flake8_checks=1; fi if [ $no_extensions_tests -ne 1 ]; then extensions_tests=1; fi @@ -208,11 +199,6 @@ function run_tests { run_upgrade_system_tests || errors+=" upgrade_system_tests" fi - if [ $shotgun_tests -eq 1 ]; then - echo "Starting Shotgun tests..." - run_shotgun_tests || errors+=" shotgun_tests" - fi - if [ $extensions_tests -eq 1 ]; then echo "Starting Extensions tests..." run_extensions_tests || errors+=" extensions_tests" @@ -377,25 +363,6 @@ function run_upgrade_system_tests { } -# Run shotgun tests -# -# Arguments: -# -# $@ -- tests to be run; with no arguments all tests will be run -function run_shotgun_tests { - local result=0 - - pushd $ROOT/shotgun >> /dev/null - - # run tests - TOXENV=$TOXENV \ - tox || result=1 - - popd >> /dev/null - - return $result -} - function run_flake8_subproject { local DIRECTORY=$1 local result=0 @@ -435,7 +402,6 @@ function run_flake8 { run_flake8_subproject network_checker && \ run_flake8_subproject fuel_upgrade_system/fuel_upgrade && \ run_flake8_subproject fuel_upgrade_system/fuel_package_updates && \ - run_flake8_subproject shotgun || result=1 return $result } @@ -619,8 +585,6 @@ function guess_test_run { run_ui_func_tests $1 elif [[ $1 == *fuel_upgrade_system* ]]; then run_upgrade_system_tests $1 - elif [[ $1 == *shotgun* ]]; then - run_shotgun_tests $1 else run_nailgun_tests $1 fi diff --git a/shotgun/.coveragerc b/shotgun/.coveragerc deleted file mode 100644 index 406be7d7f7..0000000000 --- a/shotgun/.coveragerc +++ /dev/null @@ -1,3 +0,0 @@ -[run] -branch = True -omit = shotgun/test/* diff --git a/shotgun/.testr.conf b/shotgun/.testr.conf deleted file mode 100644 index 11e26f234b..0000000000 --- a/shotgun/.testr.conf +++ /dev/null @@ -1,4 +0,0 @@ -[DEFAULT] -test_command=OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 OS_TEST_TIMEOUT=60 ${PYTHON:-python} -m subunit.run discover -s shotgun/test/ -p "*.py" $LISTOPT $IDOPTION -test_id_option=--load-list $IDFILE -test_list_option=--list diff --git a/shotgun/bin/example.json b/shotgun/bin/example.json deleted file mode 100644 index 227ed8e6a9..0000000000 --- a/shotgun/bin/example.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - - "target": "/tmp/snapshot", - "timestamp": "true", - "dump_roles": { - "master": ["localhost"], - "slave": ["srv11-msk.msk.mirantis.net"] - }, - "dump_objects": { - "master": [ - { - "type": "file", - "path": "/etc/rsyslog.d" - }, - { - "type": "postgres", - "dbname": "nailgun", - "username": "postgres" - }, - ], - "slave": [ - { - "type": "command", - "command": "ps auxww" - }, - { - "type": "command", - "command": "top" - }, - { - "type": "command", - "command": "ip a" - }, - { - "type": "file", - "path": "/etc/naily.facts" - } - ] - } -} diff --git a/shotgun/bin/example.py b/shotgun/bin/example.py deleted file mode 100644 index ccfa52f027..0000000000 --- a/shotgun/bin/example.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2013 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import json -import logging -import os -import sys - -sys.path[:0] = [os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))] - -from shotgun.config import Config -from shotgun.manager import Manager - -logging.basicConfig(level=logging.DEBUG) - - -with open("snapshot.json", "r") as fo: - data = json.loads(fo.read()) - config = Config(data) - - -manager = Manager(config) -manager.snapshot() diff --git a/shotgun/requirements.txt b/shotgun/requirements.txt deleted file mode 100644 index bdcf660b75..0000000000 --- a/shotgun/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -Fabric>=1.7.0 diff --git a/shotgun/run_tests.sh b/shotgun/run_tests.sh deleted file mode 100755 index e83b2d6701..0000000000 --- a/shotgun/run_tests.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -# Copyright 2015 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -e -set -x - -tox -v diff --git a/shotgun/setup.py b/shotgun/setup.py deleted file mode 100644 index c3681a35c2..0000000000 --- a/shotgun/setup.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2013 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import setuptools - - -setuptools.setup( - name='shotgun', - version='8.0.0', - description='Shotgun package', - long_description='Shotgun is diagnostic snapshot generator', - classifiers=[ - 'Development Status :: 4 - Beta', - 'Programming Language :: Python'], - author='Mirantis Inc.', - author_email='product@mirantis.com', - url='http://mirantis.com', - keywords='shotgun mirantis', - packages=setuptools.find_packages(), - zip_safe=False, - install_requires=[ - 'Fabric >= 1.10.0'], - entry_points={ - 'console_scripts': [ - 'shotgun = shotgun.cli:main']}) diff --git a/shotgun/shotgun/__init__.py b/shotgun/shotgun/__init__.py deleted file mode 100644 index 141ca6ab9a..0000000000 --- a/shotgun/shotgun/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2013 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. diff --git a/shotgun/shotgun/cli.py b/shotgun/shotgun/cli.py deleted file mode 100644 index 60ed4bab21..0000000000 --- a/shotgun/shotgun/cli.py +++ /dev/null @@ -1,71 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright 2014 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import argparse -import json -import logging - -from shotgun.logger import configure_logger -configure_logger() - -from shotgun.config import Config -from shotgun.manager import Manager - - -logger = logging.getLogger(__name__) - - -def parse_args(): - """Parse arguments and return them - - :returns: argparse object - """ - parser = argparse.ArgumentParser() - - parser.add_argument( - '-c', - '--config', - help='configuration file', - required=True) - return parser.parse_args() - - -def read_config(config_path): - """Reads config - - :param config_path: path to configuration file - :returns: dict with configuration data - """ - with open(config_path, "r") as fo: - config = json.loads(fo.read()) - - return config - - -def make_snapshot(args): - """Generates snapshot - - :param args: argparse object - """ - config_object = Config(read_config(args.config)) - manager = Manager(config_object) - snapshot_path = manager.snapshot() - logger.info(u'Snapshot path: {0}'.format(snapshot_path)) - - -def main(): - """Entry point""" - make_snapshot(parse_args()) diff --git a/shotgun/shotgun/config.py b/shotgun/shotgun/config.py deleted file mode 100644 index e31a6b5712..0000000000 --- a/shotgun/shotgun/config.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2013 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging -import time - -from shotgun import settings - - -logger = logging.getLogger(__name__) - - -class Config(object): - def __init__(self, data=None): - self.data = data - self.time = time.localtime() - - def _timestamp(self, name): - return "{0}-{1}".format( - name, - time.strftime('%Y-%m-%d_%H-%M-%S', self.time) - ) - - @property - def target(self): - target = self.data.get("target", settings.TARGET) - if self.data.get("timestamp", settings.TIMESTAMP): - target = self._timestamp(target) - return target - - @property - def compression_level(self): - level = self.data.get("compression_level") - if level is None: - logger.info( - 'Compression level is not specified,' - ' Default %s will be used', settings.COMPRESSION_LEVEL) - - level = settings.COMPRESSION_LEVEL - - return '-{level}'.format(level=level) - - @property - def lastdump(self): - return self.data.get("lastdump", settings.LASTDUMP) - - @property - def objects(self): - for role, properties in self.data["dump"].iteritems(): - for host in properties.get("hosts", []): - for object_ in properties.get("objects", []): - object_["host"] = host - yield object_ - - @property - def timeout(self): - """Timeout for executing commands.""" - return self.data.get("timeout", settings.DEFAULT_TIMEOUT) diff --git a/shotgun/shotgun/driver.py b/shotgun/shotgun/driver.py deleted file mode 100644 index c4f7a92f18..0000000000 --- a/shotgun/shotgun/driver.py +++ /dev/null @@ -1,257 +0,0 @@ -# Copyright 2013 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging -import os -import pprint -import pwd -import re -import stat -import sys -import xmlrpclib - -import fabric.api - -from shotgun import utils - - -logger = logging.getLogger(__name__) - - -class CommandOut(object): - stdout = None - return_code = None - stderr = None - - def __eq__(self, other): - return ( - str(self.stdout) == str(other.stdout) and - str(self.stderr) == str(other.stderr) and - str(self.return_code) == str(other.return_code) - ) - - -class Driver(object): - - @classmethod - def getDriver(cls, data, conf): - driver_type = data["type"] - return { - "file": File, - "dir": Dir, - "postgres": Postgres, - "xmlrpc": XmlRpc, - "command": Command, - }.get(driver_type, cls)(data, conf) - - def __init__(self, data, conf): - logger.debug("Initializing driver %s: host=%s", - self.__class__.__name__, data.get("host")) - self.data = data - self.host = self.data.get("host", {}).get("address", "localhost") - self.ssh_key = self.data.get("host", {}).get("ssh-key") - self.local = utils.is_local(self.host) - self.conf = conf - self.timeout = self.data.get("timeout", self.conf.timeout) - - def snapshot(self): - raise NotImplementedError - - def command(self, command): - out = CommandOut() - - raw_stdout = utils.CCStringIO(writers=sys.stdout) - try: - if not self.local: - with fabric.api.settings( - host_string=self.host, # destination host - key_filename=self.ssh_key, # a path to ssh key - timeout=2, # a network connection timeout - command_timeout=self.timeout, # command execution timeout - warn_only=True, # don't exit on error - abort_on_prompts=True, # non-interactive mode - ): - logger.debug("Running remote command: " - "host: %s command: %s", self.host, command) - try: - output = fabric.api.run(command, stdout=raw_stdout) - except SystemExit: - logger.error("Fabric aborted this iteration") - # NOTE(prmtl): because of pty=True (default) and - # combine_stderr=True (default) stderr is combined - # with stdout - out.stdout = raw_stdout.getvalue() - out.return_code = output.return_code - else: - logger.debug("Running local command: %s", command) - out.return_code, out.stdout, out.stderr = utils.execute( - command) - except Exception as e: - logger.error("Error occured: %s", str(e)) - out.stdout = raw_stdout.getvalue() - return out - - def get(self, path, target_path): - """Get remote or local file - - target_path must be the directory where to put - copied files or directories - """ - try: - if not self.local: - with fabric.api.settings( - host_string=self.host, # destination host - key_filename=self.ssh_key, # a path to ssh key - timeout=2, # a network connection timeout - warn_only=True, # don't exit on error - abort_on_prompts=True, # non-interactive mode - ): - logger.debug("Getting remote file: %s %s", - path, target_path) - utils.execute('mkdir -p "{0}"'.format(target_path)) - try: - return fabric.api.get(path, target_path) - except SystemExit: - logger.error("Fabric aborted this iteration") - else: - logger.debug("Getting local file: cp -r %s %s", - path, target_path) - utils.execute('mkdir -p "{0}"'.format(target_path)) - return utils.execute('cp -r "{0}" "{1}"'.format(path, - target_path)) - except Exception as e: - logger.error("Error occured: %s", str(e)) - - -class File(Driver): - def __init__(self, data, conf): - super(File, self).__init__(data, conf) - self.path = self.data["path"] - self.exclude = self.data.get('exclude', []) - logger.debug("File to get: %s", self.path) - self.target_path = str(os.path.join( - self.conf.target, self.host, - os.path.dirname(self.path).lstrip("/"))) - self.full_dst_path = os.path.join( - self.conf.target, self.host, - self.path.lstrip("/")) - logger.debug("File to save: %s", self.target_path) - - def snapshot(self): - """Make a snapshot - - Example: - self.conf.target IS /target - self.host IS host.domain.tld - self.path IS /var/log/somedir - self.target_path IS /target/host.domain.tld/var/log - """ - self.get(self.path, self.target_path) - - if self.exclude: - utils.remove(self.full_dst_path, self.exclude) - -Dir = File - - -class Postgres(Driver): - def __init__(self, data, conf): - super(Postgres, self).__init__(data, conf) - self.dbhost = self.data.get("dbhost", "localhost") - self.dbname = self.data["dbname"] - self.username = self.data.get("username", "postgres") - self.password = self.data.get("password") - self.target_path = str(os.path.join(self.conf.target, - self.host, "pg_dump")) - - def snapshot(self): - if self.password: - authline = "{host}:{port}:{dbname}:{username}:{password}".format( - host=self.dbhost, port="5432", dbname=self.dbname, - username=self.username, password=self.password) - home_dir = pwd.getpwuid(os.getuid()).pw_dir - pgpass = os.path.join(home_dir, ".pgpass") - with open(pgpass, "a+") as fo: - fo.seek(0) - auth = False - for line in fo: - if re.search(ur"^{0}$".format(authline), line): - auth = True - break - if not auth: - fo.seek(0, 2) - fo.write("{0}\n".format(authline)) - os.chmod(pgpass, stat.S_IRUSR + stat.S_IWUSR) - temp = self.command("mktemp").stdout.strip() - self.command("pg_dump -h {dbhost} -U {username} -w " - "-f {file} {dbname}".format( - dbhost=self.dbhost, username=self.username, - file=temp, dbname=self.dbname)) - utils.execute('mkdir -p "{0}"'.format(self.target_path)) - dump_basename = "{0}_{1}.sql".format(self.dbhost, self.dbname) - - utils.execute('mv -f "{0}" "{1}"'.format( - temp, - os.path.join(self.target_path, dump_basename))) - - -class XmlRpc(Driver): - def __init__(self, data, conf): - super(XmlRpc, self).__init__(data, conf) - - self.server = self.data.get("server", "localhost") - self.methods = self.data.get("methods", []) - self.to_file = self.data.get("to_file") - - self.target_path = os.path.join( - self.conf.target, self.host, "xmlrpc", self.to_file) - - def snapshot(self): - utils.execute('mkdir -p "{0}"'.format(os.path.dirname( - self.target_path))) - - server = xmlrpclib.Server(self.server) - with open(self.target_path, "w") as f: - for method in self.methods: - if hasattr(server, method): - response = getattr(server, method)() - response = pprint.pformat(response, indent=2) - else: - response = "no such method on remote server" - - f.write("===== {0} =====\n{1}\n\n".format(method, response)) - - -class Command(Driver): - - def __init__(self, data, conf): - super(Command, self).__init__(data, conf) - self.cmdname = self.data["command"] - self.to_file = self.data["to_file"] - self.target_path = os.path.join( - self.conf.target, self.host, "commands", self.to_file) - - def snapshot(self): - out = self.command(self.cmdname) - utils.execute('mkdir -p "{0}"'.format(os.path.dirname( - self.target_path))) - with open(self.target_path, "w") as f: - f.write("===== COMMAND =====: {0}\n".format(self.cmdname)) - f.write("===== RETURN CODE =====: {0}\n".format(out.return_code)) - f.write("===== STDOUT =====:\n") - if out.stdout: - f.write(out.stdout) - f.write("\n===== STDERR =====:\n") - if out.stderr: - f.write(out.stderr) diff --git a/shotgun/shotgun/logger.py b/shotgun/shotgun/logger.py deleted file mode 100644 index 6f43bd3ede..0000000000 --- a/shotgun/shotgun/logger.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2013 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging - -from shotgun.settings import LOG_FILE - - -def configure_logger(): - """Configures shotgun logger""" - logger = logging.getLogger('shotgun') - logger.setLevel(logging.DEBUG) - formatter = logging.Formatter( - '%(asctime)s %(levelname)s %(process)d (%(module)s) %(message)s', - "%Y-%m-%d %H:%M:%S") - - stream_handler = logging.StreamHandler() - stream_handler.setLevel(logging.DEBUG) - stream_handler.setFormatter(formatter) - - file_handler = logging.FileHandler(LOG_FILE) - file_handler.setLevel(logging.DEBUG) - file_handler.setFormatter(formatter) - - logger.addHandler(stream_handler) - logger.addHandler(file_handler) diff --git a/shotgun/shotgun/manager.py b/shotgun/shotgun/manager.py deleted file mode 100644 index 13b6dec2a2..0000000000 --- a/shotgun/shotgun/manager.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2013 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging -import os - -from shotgun.driver import Driver -from shotgun import utils - - -logger = logging.getLogger(__name__) - - -class Manager(object): - def __init__(self, conf): - logger.debug("Initializing snapshot manager") - self.conf = conf - - def snapshot(self): - logger.debug("Making snapshot") - utils.execute("rm -rf {0}".format(os.path.dirname(self.conf.target))) - for obj_data in self.conf.objects: - logger.debug("Dumping: %s", obj_data) - driver = Driver.getDriver(obj_data, self.conf) - driver.snapshot() - logger.debug("Archiving dump directory: %s", self.conf.target) - - utils.compress(self.conf.target, self.conf.compression_level) - - with open(self.conf.lastdump, "w") as fo: - fo.write("{0}.tar.xz".format(self.conf.target)) - return "{0}.tar.xz".format(self.conf.target) diff --git a/shotgun/shotgun/settings.py b/shotgun/shotgun/settings.py deleted file mode 100644 index 36661e1b0d..0000000000 --- a/shotgun/shotgun/settings.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2013 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -TARGET = "/tmp/snapshot" -LASTDUMP = "/tmp/snapshot_last" -TIMESTAMP = True -COMPRESSION_LEVEL = 3 -LOG_FILE = "/var/log/shotgun.log" -DEFAULT_TIMEOUT = 10 diff --git a/shotgun/shotgun/test/__init__.py b/shotgun/shotgun/test/__init__.py deleted file mode 100644 index 141ca6ab9a..0000000000 --- a/shotgun/shotgun/test/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2013 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. diff --git a/shotgun/shotgun/test/base.py b/shotgun/shotgun/test/base.py deleted file mode 100644 index e0565ad16b..0000000000 --- a/shotgun/shotgun/test/base.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2015 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from unittest2.case import TestCase - - -class BaseTestCase(TestCase): - """Base unit test case for shotgun tests.""" diff --git a/shotgun/shotgun/test/test_config.py b/shotgun/shotgun/test/test_config.py deleted file mode 100644 index 448e5933cd..0000000000 --- a/shotgun/shotgun/test/test_config.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2013 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import time - -import mock - -from shotgun.config import Config -from shotgun.test import base - - -class TestConfig(base.BaseTestCase): - - def test_timestamp(self): - t = time.localtime() - with mock.patch('shotgun.config.time') as MockedTime: - MockedTime.localtime.return_value = t - MockedTime.strftime.side_effect = time.strftime - conf = Config({}) - stamped = conf._timestamp("sample") - self.assertEqual( - stamped, - "sample-{0}".format(time.strftime('%Y-%m-%d_%H-%M-%S', t)) - ) - - def test_target_timestamp(self): - conf = Config({ - "target": "/tmp/sample", - "timestamp": True - }) - self.assertRegex( - conf.target, - ur"\/tmp\/sample\-[\d]{4}\-[\d]{2}\-[\d]{2}_" - "([\d]{2}\-){2}[\d]{2}", - ) - - @mock.patch('shotgun.config.settings') - def test_timeout(self, m_settings): - conf = Config({}) - self.assertIs(conf.timeout, m_settings.DEFAULT_TIMEOUT) - - def test_pass_default_timeout(self): - timeout = 1345 - conf = Config({ - 'timeout': timeout, - }) - self.assertEqual(conf.timeout, timeout) diff --git a/shotgun/shotgun/test/test_driver.py b/shotgun/shotgun/test/test_driver.py deleted file mode 100644 index 20f4d07c89..0000000000 --- a/shotgun/shotgun/test/test_driver.py +++ /dev/null @@ -1,219 +0,0 @@ -# Copyright 2013 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import random -import sys - -import fabric -import mock - -import shotgun -from shotgun.test import base - - -class RunOut(object): - return_code = None - stderr = None - stdout = None - - def __str__(self): - return str(self.stdout) - - -class TestDriver(base.BaseTestCase): - def test_driver_factory(self): - types = { - "file": "File", - "dir": "Dir", - "postgres": "Postgres", - "command": "Command" - } - for t, n in types.iteritems(): - with mock.patch("shotgun.driver.%s" % n) as mocked: - shotgun.driver.Driver.getDriver({"type": t}, None) - mocked.assert_called_with({"type": t}, None) - - @mock.patch('shotgun.driver.utils.CCStringIO') - @mock.patch('shotgun.driver.fabric.api.settings') - @mock.patch('shotgun.driver.fabric.api.run') - def test_driver_remote_command(self, mfabrun, mfabset, mccstring): - out = shotgun.driver.CommandOut() - out.stdout = "STDOUT" - out.return_code = "RETURN_CODE" - mccstring.return_value.getvalue.return_value = out.stdout - - runout = RunOut() - runout.return_code = "RETURN_CODE" - mfabrun.return_value = runout - - command = "COMMAND" - - conf = mock.Mock() - driver = shotgun.driver.Driver( - {"host": {"address": "remote_host"}}, conf) - result = driver.command(command) - - mfabrun.assert_called_with( - command, stdout=mock.ANY) - mfabset.assert_called_with( - host_string="remote_host", - timeout=2, - command_timeout=driver.timeout, - warn_only=True, - key_filename=None, - abort_on_prompts=True) - self.assertEqual(result, out) - - @mock.patch('shotgun.driver.fabric.api.run') - @mock.patch('shotgun.driver.fabric.api.settings') - def test_fabric_use_timout_from_driver(self, mfabset, _): - timeout = random.randint(1, 100) - conf = mock.Mock() - driver = shotgun.driver.Driver( - {"host": {"address": "remote_host"}}, conf) - driver.timeout = timeout - driver.command("COMMAND") - mfabset.assert_called_with( - host_string=mock.ANY, - timeout=mock.ANY, - command_timeout=timeout, - warn_only=mock.ANY, - key_filename=mock.ANY, - abort_on_prompts=mock.ANY) - - @mock.patch('shotgun.driver.utils.execute') - def test_driver_local_command(self, mexecute): - mexecute.return_value = ("RETURN_CODE", "STDOUT", "STDERR") - - out = shotgun.driver.CommandOut() - out.stdout = "STDOUT" - out.stderr = "STDERR" - out.return_code = "RETURN_CODE" - - command = "COMMAND" - conf = mock.Mock() - driver = shotgun.driver.Driver({}, conf) - result = driver.command(command) - shotgun.driver.utils.execute.assert_called_with(command) - self.assertEqual(result, out) - - @mock.patch('shotgun.driver.utils.CCStringIO') - @mock.patch('shotgun.driver.fabric.api.settings') - @mock.patch('shotgun.driver.fabric.api.run') - def test_command_timeout(self, mfabrun, mfabset, mstringio): - mfabrun.side_effect = fabric.exceptions.CommandTimeout(10) - - mstdout = mock.MagicMock() - mstdout.getvalue.return_value = 'FULL STDOUT' - mstringio.return_value = mstdout - - command = "COMMAND" - - conf = mock.Mock() - driver = shotgun.driver.Driver( - {"host": {"address": "remote_host"}}, conf) - result = driver.command(command) - - mstringio.assert_has_calls([ - mock.call(writers=sys.stdout), - ]) - mfabrun.assert_called_with(command, stdout=mstdout) - self.assertEqual(result.stdout, 'FULL STDOUT') - - @mock.patch('shotgun.driver.utils.execute') - @mock.patch('shotgun.driver.fabric.api.settings') - @mock.patch('shotgun.driver.fabric.api.get') - def test_driver_get(self, mfabget, mfabset, mexecute): - mexecute.return_value = ("RETURN_CODE", "STDOUT", "STDERR") - remote_path = "/remote_dir/remote_file" - target_path = "/target_dir" - conf = mock.Mock() - - driver = shotgun.driver.Driver({ - "host": { - "address": "remote_host", - "ssh-key": "path_to_key", - } - }, conf) - driver.get(remote_path, target_path) - mexecute.assert_called_with('mkdir -p "{0}"'.format(target_path)) - mfabget.assert_called_with(remote_path, target_path) - mfabset.assert_called_with( - host_string="remote_host", key_filename="path_to_key", - timeout=2, warn_only=True, abort_on_prompts=True) - - mexecute.reset_mock() - driver = shotgun.driver.Driver({}, conf) - driver.get(remote_path, target_path) - self.assertEqual(mexecute.mock_calls, [ - mock.call('mkdir -p "{0}"'.format(target_path)), - mock.call('cp -r "{0}" "{1}"'.format(remote_path, target_path))]) - - def test_use_timeout_from_global_conf(self): - data = {} - conf = mock.Mock(spec=shotgun.config.Config, target="some_target") - cmd_driver = shotgun.driver.Driver(data, conf) - self.assertEqual(cmd_driver.timeout, conf.timeout) - - def test_use_command_specific_timeout(self): - timeout = 1234 - data = { - "timeout": timeout - } - conf = mock.Mock(spec=shotgun.config.Config, target="some_target") - cmd_driver = shotgun.driver.Driver(data, conf) - self.assertEqual(cmd_driver.timeout, timeout) - self.assertNotEqual(cmd_driver.timeout, conf.timeout) - - -class TestFile(base.BaseTestCase): - - @mock.patch('shotgun.driver.Driver.get') - def test_snapshot(self, mget): - data = { - "type": "file", - "path": "/remote_dir/remote_file", - "host": { - "address": "remote_host", - }, - } - conf = mock.MagicMock() - conf.target = "/target" - file_driver = shotgun.driver.File(data, conf) - - target_path = "/target/remote_host/remote_dir" - file_driver.snapshot() - - mget.assert_called_with(data["path"], target_path) - - @mock.patch('shotgun.driver.utils.remove') - @mock.patch('shotgun.driver.Driver.get') - def test_dir_exclude_called(self, mget, mremove): - data = { - "type": "dir", - "path": "/remote_dir/", - "exclude": ["*test"], - "host": { - "address": "remote_host", - }, - } - conf = mock.MagicMock() - conf.target = "/target" - dir_driver = shotgun.driver.Dir(data, conf) - - target_path = "/target/remote_host/remote_dir" - dir_driver.snapshot() - - mget.assert_called_with(data["path"], target_path) - mremove.assert_called_with(dir_driver.full_dst_path, data['exclude']) diff --git a/shotgun/shotgun/test/test_manager.py b/shotgun/shotgun/test/test_manager.py deleted file mode 100644 index 1d08dc90ac..0000000000 --- a/shotgun/shotgun/test/test_manager.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2015 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import tempfile - -import mock - -from shotgun.manager import Manager -from shotgun.test import base - - -class TestManager(base.BaseTestCase): - - @mock.patch('shotgun.manager.Driver.getDriver') - @mock.patch('shotgun.manager.utils.execute') - @mock.patch('shotgun.manager.utils.compress') - def test_snapshot(self, mcompress, mexecute, mget): - data = { - "type": "file", - "path": "/remote_dir/remote_file", - "host": { - "address": "remote_host", - }, - } - conf = mock.MagicMock() - conf.target = "/target/data" - conf.objects = [data] - conf.lastdump = tempfile.mkstemp()[1] - manager = Manager(conf) - manager.snapshot() - mget.assert_called_once_with(data, conf) - mexecute.assert_called_once_with('rm -rf /target') diff --git a/shotgun/shotgun/test/test_utils.py b/shotgun/shotgun/test/test_utils.py deleted file mode 100644 index 573fb97ae5..0000000000 --- a/shotgun/shotgun/test/test_utils.py +++ /dev/null @@ -1,112 +0,0 @@ -# coding: utf-8 - -# Copyright 2015 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import StringIO - -import mock - -from shotgun.test import base -from shotgun import utils - - -class TestUtils(base.BaseTestCase): - - @mock.patch('shotgun.utils.execute') - def test_remove_subdir(self, mexecute): - utils.remove('/', ['good', '**/*.py']) - mexecute.assert_has_calls([ - mock.call('shopt -s globstar; rm -rf /good', shell=True), - mock.call('shopt -s globstar; rm -rf /**/*.py', shell=True)]) - - @mock.patch('shotgun.utils.os.walk') - def test_iterfiles(self, mwalk): - path = '/root' - mwalk.return_value = [ - (path, '', ('file1', 'file2')), - (path + '/sub', '', ('file3',))] - - result = list(utils.iterfiles(path)) - - mwalk.assert_called_once_with(path, topdown=True) - self.assertEqual( - result, ['/root/file1', '/root/file2', '/root/sub/file3']) - - @mock.patch('shotgun.utils.execute') - def test_compress(self, mexecute): - target = '/path/target' - level = '-3' - - utils.compress(target, level) - - compress_call = mexecute.call_args_list[0] - rm_call = mexecute.call_args_list[1] - - compress_env = compress_call[1]['env'] - self.assertEqual(compress_env['XZ_OPT'], level) - self.assertEqual( - compress_call[0][0], - 'tar cJvf /path/target.tar.xz -C /path target') - - self.assertEqual(rm_call[0][0], 'rm -r /path/target') - - -class TestCCStringIO(base.BaseTestCase): - - def test_no_writers(self): - test_string = 'some_string' - - ccstring = utils.CCStringIO() - ccstring.write(test_string) - - self.assertEqual(ccstring.getvalue(), test_string) - - def test_with_one_writer(self): - test_string = 'some_string' - - writer = StringIO.StringIO() - ccstring = utils.CCStringIO(writers=writer) - ccstring.write(test_string) - - self.assertEqual(ccstring.getvalue(), test_string) - self.assertEqual(writer.getvalue(), test_string) - - def test_with_multiple_writers(self): - test_string = 'some_string' - - writer_a = StringIO.StringIO() - writer_b = StringIO.StringIO() - ccstring = utils.CCStringIO(writers=[writer_a, writer_b]) - ccstring.write(test_string) - - self.assertEqual(ccstring.getvalue(), test_string) - self.assertEqual(writer_a.getvalue(), test_string) - self.assertEqual(writer_b.getvalue(), test_string) - - def test_with_writer_and_buffer(self): - buffer = 'I am here already' - - writer = StringIO.StringIO() - ccstring = utils.CCStringIO(buffer, writers=writer) - - self.assertEqual(ccstring.getvalue(), buffer) - self.assertEqual(writer.getvalue(), '') - - def test_non_ascii_output_with_unicode(self): - ccstring = utils.CCStringIO() - ccstring.write('привет') - ccstring.write(u'test') - - self.assertEqual(ccstring.getvalue(), 'приветtest') diff --git a/shotgun/shotgun/utils.py b/shotgun/shotgun/utils.py deleted file mode 100644 index 2b8dde7e1e..0000000000 --- a/shotgun/shotgun/utils.py +++ /dev/null @@ -1,158 +0,0 @@ -# Copyright 2013 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import copy -import logging -import os -import re -import shlex -import socket -from StringIO import StringIO -import subprocess - - -logger = logging.getLogger(__name__) - - -def hostname(): - return socket.gethostname() - - -def is_ip(name): - return (re.search(ur"([0-9]{1,3}\.){3}[0-9]{1,3}", name) and True) - - -def fqdn(name=None): - if name: - return socket.getfqdn(name) - return socket.getfqdn(socket.gethostname()) - - -def is_local(name): - if name in ("localhost", hostname(), fqdn()): - return True - return False - - -def iterfiles(path): - for root, dirnames, filenames in os.walk(path, topdown=True): - for filename in filenames: - yield os.path.join(root, filename) - - -def remove(full_dst_path, excludes): - """Removes subdirs/files using unixs syntax - - full_dst_path is treated as root directory for remove - - :param full_dst_path: str - :param excludes: list with excludes paths/files - """ - for exclude in excludes: - path = os.path.join(full_dst_path, exclude.lstrip('/')) - logger.debug('Deleting %s', path) - execute("shopt -s globstar; rm -rf {0}".format(path), shell=True) - - -def compress(target, level, keep_target=False): - """Runs compression of provided directory - - :param target: directory to compress - :param level: level of compression - :param keep_target: bool, if True target directory wont be removed - """ - env = copy.deepcopy(os.environ) - env['XZ_OPT'] = level - execute("tar cJvf {0}.tar.xz -C {1} {2}" - "".format(target, - os.path.dirname(target), - os.path.basename(target)), - env=env) - if not keep_target: - execute("rm -r {0}".format(target)) - - -def execute(command, to_filename=None, env=None, shell=False): - logger.debug("Trying to execute command: %s", command) - commands = [c.strip() for c in re.split(ur'\|', command)] - env = env or os.environ - env["PATH"] = "/bin:/usr/bin:/sbin:/usr/sbin" - - to_file = None - if to_filename: - to_file = open(to_filename, 'wb') - - process = [] - for c in commands: - try: - # NOTE(eli): Python's shlex implementation doesn't like unicode. - # We have to convert to ascii before shlex'ing the command. - # http://bugs.python.org/issue6988 - encoded_command = c.encode('ascii') - process.append(subprocess.Popen( - shlex.split(encoded_command) if not shell else encoded_command, - env=env, - stdin=(process[-1].stdout if process else None), - stdout=(to_file - if (len(process) == len(commands) - 1) and to_file - else subprocess.PIPE), - stderr=(subprocess.PIPE), - shell=shell - )) - except OSError as e: - return (1, "", "{0}\n".format(e)) - - if len(process) >= 2: - process[-2].stdout.close() - stdout, stderr = process[-1].communicate() - return (process[-1].returncode, stdout, stderr) - - -class CCStringIO(StringIO): - """A "carbon copy" StringIO. - - It's capable of multiplexing its writes to other buffer objects. - - Taken from fabric.tests.mock_streams.CarbonCopy - """ - - def __init__(self, buffer='', writers=None): - """CCStringIO initializator - - If ``writers`` is given and is a file-like object or an - iterable of same, it/they will be written to whenever this - StringIO instance is written to. - """ - StringIO.__init__(self, buffer) - if writers is None: - writers = [] - elif hasattr(writers, 'write'): - writers = [writers] - self.writers = writers - - def write(self, s): - # unfortunately, fabric writes into StringIO both so-called - # bytestrings and unicode strings. obviously, bytestrings may - # contain non-ascii symbols. that leads to type-conversion - # issue when we use string's join (inside getvalue()) with - # a list of both unicodes and bytestrings. in order to avoid - # this issue we should convert all input unicode strings into - # utf-8 bytestrings (let's assume that slaves encoding is utf-8 - # too so we won't have encoding mess in the output file). - if isinstance(s, unicode): - s = s.encode('utf-8') - - StringIO.write(self, s) - for writer in self.writers: - writer.write(s) diff --git a/shotgun/specs/shotgun.spec b/shotgun/specs/shotgun.spec deleted file mode 100644 index bdc578bde6..0000000000 --- a/shotgun/specs/shotgun.spec +++ /dev/null @@ -1,40 +0,0 @@ -%define name shotgun -%{!?version: %define version 8.0.0} -%{!?release: %define release 1} - -Name: %{name} -Summary: Shotgun package -Version: %{version} -Release: %{release} -URL: http://mirantis.com -License: Apache -Group: Development/Libraries -BuildRoot: %{_tmppath}/%{name}-%{version}-buildroot -Prefix: %{_prefix} -BuildArch: noarch -Requires: postgresql -Requires: python-fabric >= 1.10.0 -Requires: python-argparse -Requires: tar -Requires: gzip -Requires: bzip2 -Requires: openssh-clients -Requires: xz - -%description -Shotgun package. - -%prep -%setup -cq -n %{name}-%{version} - -%build -cd %{_builddir}/%{name}-%{version}/shotgun && python setup.py build - -%install -cd %{_builddir}/%{name}-%{version}/shotgun && python setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=%{_builddir}/%{name}-%{version}/shotgun/INSTALLED_FILES - -%clean -rm -rf $RPM_BUILD_ROOT - -%files -f %{_builddir}/%{name}-%{version}/shotgun/INSTALLED_FILES -%defattr(-,root,root) diff --git a/shotgun/test-requirements.txt b/shotgun/test-requirements.txt deleted file mode 100644 index 77b5549c3f..0000000000 --- a/shotgun/test-requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -mock==1.3.0 -unittest2==1.1.0 -pytest==2.8.0 -pytest-cov==2.1.0 diff --git a/shotgun/tox.ini b/shotgun/tox.ini deleted file mode 100644 index 8344def361..0000000000 --- a/shotgun/tox.ini +++ /dev/null @@ -1,43 +0,0 @@ -[tox] -minversion = 1.6 -skipsdist = True -envlist = py26,py27,pep8 - -[testenv] -usedevelop = True -install_command = pip install --allow-external -U {opts} {packages} -setenv = VIRTUAL_ENV={envdir} -deps = -r{toxinidir}/test-requirements.txt -commands = - py.test -vv {posargs:shotgun/test} - -[tox:jenkins] -downloadcache = ~/cache/pip - -[testenv:pep8] -deps = hacking==0.10 -usedevelop = False -commands = - flake8 {posargs:shotgun} - -[testenv:cover] -setenv = VIRTUAL_ENV={envdir} -commands = - py.test -vv --cov=shotgun {posargs:shotgun/test} - -[testenv:venv] -commands = {posargs:} - -[testenv:devenv] -envdir = devenv -usedevelop = True - -[flake8] -ignore = H234,H302,H802 -exclude = .venv,.git,.tox,dist,doc,*lib/python*,*egg,build,tools,__init__.py,docs -show-pep8 = True -show-source = True -count = True - -[hacking] -import_exceptions = testtools.matchers diff --git a/specs/fuel-nailgun.spec b/specs/fuel-nailgun.spec index 1a1d53d9f5..af8df1c0ef 100644 --- a/specs/fuel-nailgun.spec +++ b/specs/fuel-nailgun.spec @@ -117,13 +117,11 @@ cd %{_builddir}/%{name}-%{version}/nailgun && %{_builddir}/%{name}-%{version}/na mv %{_builddir}/%{name}-%{version}/nailgun/compressed_static %{_builddir}/%{name}-%{version}/nailgun/static cd %{_builddir}/%{name}-%{version}/nailgun && python setup.py build cd %{_builddir}/%{name}-%{version}/network_checker && python setup.py build -cd %{_builddir}/%{name}-%{version}/shotgun && python setup.py build cd %{_builddir}/%{name}-%{version}/fuel_upgrade_system/fuel_package_updates && python setup.py build %install cd %{_builddir}/%{name}-%{version}/nailgun && python setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=%{_builddir}/%{name}-%{version}/nailgun/INSTALLED_FILES cd %{_builddir}/%{name}-%{version}/network_checker && python setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=%{_builddir}/%{name}-%{version}/network_checker/INSTALLED_FILES -cd %{_builddir}/%{name}-%{version}/shotgun && python setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=%{_builddir}/%{name}-%{version}/shotgun/INSTALLED_FILES cd %{_builddir}/%{name}-%{version}/fuel_upgrade_system/fuel_package_updates && python setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=%{_builddir}/%{name}-%{version}/fuel_upgrade_system/fuel_package_updates/INSTALLED_FILES mkdir -p %{buildroot}/opt/nailgun/bin mkdir -p %{buildroot}/etc/cron.d @@ -185,31 +183,6 @@ between hosts in network. %files -n nailgun-net-check -f %{_builddir}/%{name}-%{version}/network_checker/INSTALLED_FILES %defattr(-,root,root) -%package -n shotgun - -Summary: Shotgun package -Version: %{version} -Release: %{release} -URL: http://mirantis.com -License: Apache -Group: Development/Libraries -BuildRoot: %{_tmppath}/%{name}-%{version}-buildroot -Prefix: %{_prefix} -BuildArch: noarch -Requires: postgresql -Requires: python-fabric >= 1.10.0 -Requires: python-argparse -Requires: tar -Requires: gzip -Requires: bzip2 -Requires: openssh-clients -Requires: xz - -%description -n shotgun -Shotgun package. - -%files -n shotgun -f %{_builddir}/%{name}-%{version}/shotgun/INSTALLED_FILES -%defattr(-,root,root) %package -n fencing-agent Summary: Fencing agent