Deprecate shotgun directory
Change-Id: Ib1260df78a45e9cc8bf6bdfd9627508ce50ae812 Related-Bug: #1506894
This commit is contained in:
parent
820a096c3a
commit
65b07097b2
|
@ -49,15 +49,6 @@ maintainers:
|
|||
email: loles@mirantis.com
|
||||
IRC: salmon_
|
||||
|
||||
- shotgun/:
|
||||
- name: Maciej Kwiek
|
||||
email: mkwiek@mirantis.com
|
||||
IRC: mkwiek
|
||||
|
||||
- name: Vladimir Kozhukalov
|
||||
email: vkozhukalov@mirantis.com
|
||||
IRC: kozhukalov
|
||||
|
||||
- nailgun/:
|
||||
- name: Aleksandr Kislitskii
|
||||
email: akislitsky@mirantis.com
|
||||
|
|
|
@ -88,7 +88,6 @@ prepare_venv() {
|
|||
virtualenv "${VENV}" # you can use any name instead of 'fuel'
|
||||
. "${VENV}/bin/activate" # command selects the particular environment
|
||||
# install dependencies
|
||||
pip install ./shotgun # this fuel project is listed in setup.py requirements
|
||||
pip install -r 'nailgun/test-requirements.txt'
|
||||
}
|
||||
|
||||
|
|
|
@ -273,7 +273,6 @@ Now you can create the virtual environment and activate it.
|
|||
And then install the dependencies.
|
||||
::
|
||||
|
||||
pip install ./shotgun
|
||||
pip install -r nailgun/test-requirements.txt
|
||||
|
||||
Now you can look at the list of available formats and generate
|
||||
|
|
|
@ -78,7 +78,6 @@ Preparing Development Environment
|
|||
sudo apt-get install --yes git
|
||||
git clone https://github.com/openstack/fuel-web.git
|
||||
cd fuel-web
|
||||
pip install ./shotgun # this fuel project is listed in setup.py requirements
|
||||
pip install --allow-all-external -r nailgun/test-requirements.txt
|
||||
|
||||
#. Create required folder for log files::
|
||||
|
@ -302,4 +301,3 @@ you could only run dropdb with *./run_tests.sh* script.
|
|||
Now you need to run dropdb for each slave node:
|
||||
the *py.test --cleandb <path to the tests>* command is introduced for this
|
||||
purpose.
|
||||
|
||||
|
|
36
run_tests.sh
36
run_tests.sh
|
@ -26,8 +26,6 @@ function usage {
|
|||
echo " -x, --performance Run NAILGUN performance tests"
|
||||
echo " -p, --flake8 Run FLAKE8 and HACKING compliance check"
|
||||
echo " -P, --no-flake8 Don't run static code checks"
|
||||
echo " -s, --shotgun Run SHOTGUN tests"
|
||||
echo " -S, --no-shotgun Don't run SHOTGUN tests"
|
||||
echo " -t, --tests Run a given test files"
|
||||
echo " -u, --upgrade Run tests for UPGRADE system"
|
||||
echo " -U, --no-upgrade Don't run tests for UPGRADE system"
|
||||
|
@ -59,8 +57,6 @@ function process_options {
|
|||
-x|--performance) performance_tests=1;;
|
||||
-u|--upgrade) upgrade_system=1;;
|
||||
-U|--no-upgrade) no_upgrade_system=1;;
|
||||
-s|--shotgun) shotgun_tests=1;;
|
||||
-S|--no-shotgun) no_shotgun_tests=1;;
|
||||
-p|--flake8) flake8_checks=1;;
|
||||
-P|--no-flake8) no_flake8_checks=1;;
|
||||
-w|--webui) ui_lint_checks=1; ui_unit_tests=1; ui_func_tests=1;;
|
||||
|
@ -100,7 +96,6 @@ testropts="--with-timer --timer-warning=10 --timer-ok=2 --timer-top-n=10"
|
|||
# nosetest xunit options
|
||||
NAILGUN_XUNIT=${NAILGUN_XUNIT:-"$ROOT/nailgun.xml"}
|
||||
FUELUPGRADE_XUNIT=${FUELUPGRADE_XUNIT:-"$ROOT/fuelupgrade.xml"}
|
||||
SHOTGUN_XUNIT=${SHOTGUN_XUNIT:-"$ROOT/shotgun.xml"}
|
||||
EXTENSIONS_XUNIT=${EXTENSIONS_XUNIT:-"$ROOT/extensions.xml"}
|
||||
NAILGUN_PORT=${NAILGUN_PORT:-5544}
|
||||
TEST_NAILGUN_DB=${TEST_NAILGUN_DB:-nailgun}
|
||||
|
@ -118,8 +113,6 @@ no_nailgun_tests=0
|
|||
performance_tests=0
|
||||
upgrade_system=0
|
||||
no_upgrade_system=0
|
||||
shotgun_tests=0
|
||||
no_shotgun_tests=0
|
||||
flake8_checks=0
|
||||
no_flake8_checks=0
|
||||
ui_lint_checks=0
|
||||
|
@ -162,7 +155,6 @@ function run_tests {
|
|||
$ui_unit_tests -eq 0 && \
|
||||
$ui_func_tests -eq 0 && \
|
||||
$upgrade_system -eq 0 && \
|
||||
$shotgun_tests -eq 0 && \
|
||||
$extensions_tests -eq 0 && \
|
||||
$flake8_checks -eq 0 ]]; then
|
||||
|
||||
|
@ -171,7 +163,6 @@ function run_tests {
|
|||
if [ $no_ui_unit_tests -ne 1 ]; then ui_unit_tests=1; fi
|
||||
if [ $no_ui_func_tests -ne 1 ]; then ui_func_tests=1; fi
|
||||
if [ $no_upgrade_system -ne 1 ]; then upgrade_system=1; fi
|
||||
if [ $no_shotgun_tests -ne 1 ]; then shotgun_tests=1; fi
|
||||
if [ $no_flake8_checks -ne 1 ]; then flake8_checks=1; fi
|
||||
if [ $no_extensions_tests -ne 1 ]; then extensions_tests=1; fi
|
||||
|
||||
|
@ -208,11 +199,6 @@ function run_tests {
|
|||
run_upgrade_system_tests || errors+=" upgrade_system_tests"
|
||||
fi
|
||||
|
||||
if [ $shotgun_tests -eq 1 ]; then
|
||||
echo "Starting Shotgun tests..."
|
||||
run_shotgun_tests || errors+=" shotgun_tests"
|
||||
fi
|
||||
|
||||
if [ $extensions_tests -eq 1 ]; then
|
||||
echo "Starting Extensions tests..."
|
||||
run_extensions_tests || errors+=" extensions_tests"
|
||||
|
@ -377,25 +363,6 @@ function run_upgrade_system_tests {
|
|||
}
|
||||
|
||||
|
||||
# Run shotgun tests
|
||||
#
|
||||
# Arguments:
|
||||
#
|
||||
# $@ -- tests to be run; with no arguments all tests will be run
|
||||
function run_shotgun_tests {
|
||||
local result=0
|
||||
|
||||
pushd $ROOT/shotgun >> /dev/null
|
||||
|
||||
# run tests
|
||||
TOXENV=$TOXENV \
|
||||
tox || result=1
|
||||
|
||||
popd >> /dev/null
|
||||
|
||||
return $result
|
||||
}
|
||||
|
||||
function run_flake8_subproject {
|
||||
local DIRECTORY=$1
|
||||
local result=0
|
||||
|
@ -435,7 +402,6 @@ function run_flake8 {
|
|||
run_flake8_subproject network_checker && \
|
||||
run_flake8_subproject fuel_upgrade_system/fuel_upgrade && \
|
||||
run_flake8_subproject fuel_upgrade_system/fuel_package_updates && \
|
||||
run_flake8_subproject shotgun || result=1
|
||||
return $result
|
||||
}
|
||||
|
||||
|
@ -619,8 +585,6 @@ function guess_test_run {
|
|||
run_ui_func_tests $1
|
||||
elif [[ $1 == *fuel_upgrade_system* ]]; then
|
||||
run_upgrade_system_tests $1
|
||||
elif [[ $1 == *shotgun* ]]; then
|
||||
run_shotgun_tests $1
|
||||
else
|
||||
run_nailgun_tests $1
|
||||
fi
|
||||
|
|
|
@ -1,3 +0,0 @@
|
|||
[run]
|
||||
branch = True
|
||||
omit = shotgun/test/*
|
|
@ -1,4 +0,0 @@
|
|||
[DEFAULT]
|
||||
test_command=OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 OS_TEST_TIMEOUT=60 ${PYTHON:-python} -m subunit.run discover -s shotgun/test/ -p "*.py" $LISTOPT $IDOPTION
|
||||
test_id_option=--load-list $IDFILE
|
||||
test_list_option=--list
|
|
@ -1,40 +0,0 @@
|
|||
{
|
||||
|
||||
"target": "/tmp/snapshot",
|
||||
"timestamp": "true",
|
||||
"dump_roles": {
|
||||
"master": ["localhost"],
|
||||
"slave": ["srv11-msk.msk.mirantis.net"]
|
||||
},
|
||||
"dump_objects": {
|
||||
"master": [
|
||||
{
|
||||
"type": "file",
|
||||
"path": "/etc/rsyslog.d"
|
||||
},
|
||||
{
|
||||
"type": "postgres",
|
||||
"dbname": "nailgun",
|
||||
"username": "postgres"
|
||||
},
|
||||
],
|
||||
"slave": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "ps auxww"
|
||||
},
|
||||
{
|
||||
"type": "command",
|
||||
"command": "top"
|
||||
},
|
||||
{
|
||||
"type": "command",
|
||||
"command": "ip a"
|
||||
},
|
||||
{
|
||||
"type": "file",
|
||||
"path": "/etc/naily.facts"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
|
@ -1,34 +0,0 @@
|
|||
# Copyright 2013 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path[:0] = [os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))]
|
||||
|
||||
from shotgun.config import Config
|
||||
from shotgun.manager import Manager
|
||||
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
|
||||
|
||||
with open("snapshot.json", "r") as fo:
|
||||
data = json.loads(fo.read())
|
||||
config = Config(data)
|
||||
|
||||
|
||||
manager = Manager(config)
|
||||
manager.snapshot()
|
|
@ -1 +0,0 @@
|
|||
Fabric>=1.7.0
|
|
@ -1,20 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
tox -v
|
|
@ -1,36 +0,0 @@
|
|||
# Copyright 2013 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import setuptools
|
||||
|
||||
|
||||
setuptools.setup(
|
||||
name='shotgun',
|
||||
version='8.0.0',
|
||||
description='Shotgun package',
|
||||
long_description='Shotgun is diagnostic snapshot generator',
|
||||
classifiers=[
|
||||
'Development Status :: 4 - Beta',
|
||||
'Programming Language :: Python'],
|
||||
author='Mirantis Inc.',
|
||||
author_email='product@mirantis.com',
|
||||
url='http://mirantis.com',
|
||||
keywords='shotgun mirantis',
|
||||
packages=setuptools.find_packages(),
|
||||
zip_safe=False,
|
||||
install_requires=[
|
||||
'Fabric >= 1.10.0'],
|
||||
entry_points={
|
||||
'console_scripts': [
|
||||
'shotgun = shotgun.cli:main']})
|
|
@ -1,13 +0,0 @@
|
|||
# Copyright 2013 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
|
@ -1,71 +0,0 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
|
||||
from shotgun.logger import configure_logger
|
||||
configure_logger()
|
||||
|
||||
from shotgun.config import Config
|
||||
from shotgun.manager import Manager
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def parse_args():
|
||||
"""Parse arguments and return them
|
||||
|
||||
:returns: argparse object
|
||||
"""
|
||||
parser = argparse.ArgumentParser()
|
||||
|
||||
parser.add_argument(
|
||||
'-c',
|
||||
'--config',
|
||||
help='configuration file',
|
||||
required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def read_config(config_path):
|
||||
"""Reads config
|
||||
|
||||
:param config_path: path to configuration file
|
||||
:returns: dict with configuration data
|
||||
"""
|
||||
with open(config_path, "r") as fo:
|
||||
config = json.loads(fo.read())
|
||||
|
||||
return config
|
||||
|
||||
|
||||
def make_snapshot(args):
|
||||
"""Generates snapshot
|
||||
|
||||
:param args: argparse object
|
||||
"""
|
||||
config_object = Config(read_config(args.config))
|
||||
manager = Manager(config_object)
|
||||
snapshot_path = manager.snapshot()
|
||||
logger.info(u'Snapshot path: {0}'.format(snapshot_path))
|
||||
|
||||
|
||||
def main():
|
||||
"""Entry point"""
|
||||
make_snapshot(parse_args())
|
|
@ -1,69 +0,0 @@
|
|||
# Copyright 2013 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import logging
|
||||
import time
|
||||
|
||||
from shotgun import settings
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Config(object):
|
||||
def __init__(self, data=None):
|
||||
self.data = data
|
||||
self.time = time.localtime()
|
||||
|
||||
def _timestamp(self, name):
|
||||
return "{0}-{1}".format(
|
||||
name,
|
||||
time.strftime('%Y-%m-%d_%H-%M-%S', self.time)
|
||||
)
|
||||
|
||||
@property
|
||||
def target(self):
|
||||
target = self.data.get("target", settings.TARGET)
|
||||
if self.data.get("timestamp", settings.TIMESTAMP):
|
||||
target = self._timestamp(target)
|
||||
return target
|
||||
|
||||
@property
|
||||
def compression_level(self):
|
||||
level = self.data.get("compression_level")
|
||||
if level is None:
|
||||
logger.info(
|
||||
'Compression level is not specified,'
|
||||
' Default %s will be used', settings.COMPRESSION_LEVEL)
|
||||
|
||||
level = settings.COMPRESSION_LEVEL
|
||||
|
||||
return '-{level}'.format(level=level)
|
||||
|
||||
@property
|
||||
def lastdump(self):
|
||||
return self.data.get("lastdump", settings.LASTDUMP)
|
||||
|
||||
@property
|
||||
def objects(self):
|
||||
for role, properties in self.data["dump"].iteritems():
|
||||
for host in properties.get("hosts", []):
|
||||
for object_ in properties.get("objects", []):
|
||||
object_["host"] = host
|
||||
yield object_
|
||||
|
||||
@property
|
||||
def timeout(self):
|
||||
"""Timeout for executing commands."""
|
||||
return self.data.get("timeout", settings.DEFAULT_TIMEOUT)
|
|
@ -1,257 +0,0 @@
|
|||
# Copyright 2013 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import logging
|
||||
import os
|
||||
import pprint
|
||||
import pwd
|
||||
import re
|
||||
import stat
|
||||
import sys
|
||||
import xmlrpclib
|
||||
|
||||
import fabric.api
|
||||
|
||||
from shotgun import utils
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CommandOut(object):
|
||||
stdout = None
|
||||
return_code = None
|
||||
stderr = None
|
||||
|
||||
def __eq__(self, other):
|
||||
return (
|
||||
str(self.stdout) == str(other.stdout) and
|
||||
str(self.stderr) == str(other.stderr) and
|
||||
str(self.return_code) == str(other.return_code)
|
||||
)
|
||||
|
||||
|
||||
class Driver(object):
|
||||
|
||||
@classmethod
|
||||
def getDriver(cls, data, conf):
|
||||
driver_type = data["type"]
|
||||
return {
|
||||
"file": File,
|
||||
"dir": Dir,
|
||||
"postgres": Postgres,
|
||||
"xmlrpc": XmlRpc,
|
||||
"command": Command,
|
||||
}.get(driver_type, cls)(data, conf)
|
||||
|
||||
def __init__(self, data, conf):
|
||||
logger.debug("Initializing driver %s: host=%s",
|
||||
self.__class__.__name__, data.get("host"))
|
||||
self.data = data
|
||||
self.host = self.data.get("host", {}).get("address", "localhost")
|
||||
self.ssh_key = self.data.get("host", {}).get("ssh-key")
|
||||
self.local = utils.is_local(self.host)
|
||||
self.conf = conf
|
||||
self.timeout = self.data.get("timeout", self.conf.timeout)
|
||||
|
||||
def snapshot(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def command(self, command):
|
||||
out = CommandOut()
|
||||
|
||||
raw_stdout = utils.CCStringIO(writers=sys.stdout)
|
||||
try:
|
||||
if not self.local:
|
||||
with fabric.api.settings(
|
||||
host_string=self.host, # destination host
|
||||
key_filename=self.ssh_key, # a path to ssh key
|
||||
timeout=2, # a network connection timeout
|
||||
command_timeout=self.timeout, # command execution timeout
|
||||
warn_only=True, # don't exit on error
|
||||
abort_on_prompts=True, # non-interactive mode
|
||||
):
|
||||
logger.debug("Running remote command: "
|
||||
"host: %s command: %s", self.host, command)
|
||||
try:
|
||||
output = fabric.api.run(command, stdout=raw_stdout)
|
||||
except SystemExit:
|
||||
logger.error("Fabric aborted this iteration")
|
||||
# NOTE(prmtl): because of pty=True (default) and
|
||||
# combine_stderr=True (default) stderr is combined
|
||||
# with stdout
|
||||
out.stdout = raw_stdout.getvalue()
|
||||
out.return_code = output.return_code
|
||||
else:
|
||||
logger.debug("Running local command: %s", command)
|
||||
out.return_code, out.stdout, out.stderr = utils.execute(
|
||||
command)
|
||||
except Exception as e:
|
||||
logger.error("Error occured: %s", str(e))
|
||||
out.stdout = raw_stdout.getvalue()
|
||||
return out
|
||||
|
||||
def get(self, path, target_path):
|
||||
"""Get remote or local file
|
||||
|
||||
target_path must be the directory where to put
|
||||
copied files or directories
|
||||
"""
|
||||
try:
|
||||
if not self.local:
|
||||
with fabric.api.settings(
|
||||
host_string=self.host, # destination host
|
||||
key_filename=self.ssh_key, # a path to ssh key
|
||||
timeout=2, # a network connection timeout
|
||||
warn_only=True, # don't exit on error
|
||||
abort_on_prompts=True, # non-interactive mode
|
||||
):
|
||||
logger.debug("Getting remote file: %s %s",
|
||||
path, target_path)
|
||||
utils.execute('mkdir -p "{0}"'.format(target_path))
|
||||
try:
|
||||
return fabric.api.get(path, target_path)
|
||||
except SystemExit:
|
||||
logger.error("Fabric aborted this iteration")
|
||||
else:
|
||||
logger.debug("Getting local file: cp -r %s %s",
|
||||
path, target_path)
|
||||
utils.execute('mkdir -p "{0}"'.format(target_path))
|
||||
return utils.execute('cp -r "{0}" "{1}"'.format(path,
|
||||
target_path))
|
||||
except Exception as e:
|
||||
logger.error("Error occured: %s", str(e))
|
||||
|
||||
|
||||
class File(Driver):
|
||||
def __init__(self, data, conf):
|
||||
super(File, self).__init__(data, conf)
|
||||
self.path = self.data["path"]
|
||||
self.exclude = self.data.get('exclude', [])
|
||||
logger.debug("File to get: %s", self.path)
|
||||
self.target_path = str(os.path.join(
|
||||
self.conf.target, self.host,
|
||||
os.path.dirname(self.path).lstrip("/")))
|
||||
self.full_dst_path = os.path.join(
|
||||
self.conf.target, self.host,
|
||||
self.path.lstrip("/"))
|
||||
logger.debug("File to save: %s", self.target_path)
|
||||
|
||||
def snapshot(self):
|
||||
"""Make a snapshot
|
||||
|
||||
Example:
|
||||
self.conf.target IS /target
|
||||
self.host IS host.domain.tld
|
||||
self.path IS /var/log/somedir
|
||||
self.target_path IS /target/host.domain.tld/var/log
|
||||
"""
|
||||
self.get(self.path, self.target_path)
|
||||
|
||||
if self.exclude:
|
||||
utils.remove(self.full_dst_path, self.exclude)
|
||||
|
||||
Dir = File
|
||||
|
||||
|
||||
class Postgres(Driver):
|
||||
def __init__(self, data, conf):
|
||||
super(Postgres, self).__init__(data, conf)
|
||||
self.dbhost = self.data.get("dbhost", "localhost")
|
||||
self.dbname = self.data["dbname"]
|
||||
self.username = self.data.get("username", "postgres")
|
||||
self.password = self.data.get("password")
|
||||
self.target_path = str(os.path.join(self.conf.target,
|
||||
self.host, "pg_dump"))
|
||||
|
||||
def snapshot(self):
|
||||
if self.password:
|
||||
authline = "{host}:{port}:{dbname}:{username}:{password}".format(
|
||||
host=self.dbhost, port="5432", dbname=self.dbname,
|
||||
username=self.username, password=self.password)
|
||||
home_dir = pwd.getpwuid(os.getuid()).pw_dir
|
||||
pgpass = os.path.join(home_dir, ".pgpass")
|
||||
with open(pgpass, "a+") as fo:
|
||||
fo.seek(0)
|
||||
auth = False
|
||||
for line in fo:
|
||||
if re.search(ur"^{0}$".format(authline), line):
|
||||
auth = True
|
||||
break
|
||||
if not auth:
|
||||
fo.seek(0, 2)
|
||||
fo.write("{0}\n".format(authline))
|
||||
os.chmod(pgpass, stat.S_IRUSR + stat.S_IWUSR)
|
||||
temp = self.command("mktemp").stdout.strip()
|
||||
self.command("pg_dump -h {dbhost} -U {username} -w "
|
||||
"-f {file} {dbname}".format(
|
||||
dbhost=self.dbhost, username=self.username,
|
||||
file=temp, dbname=self.dbname))
|
||||
utils.execute('mkdir -p "{0}"'.format(self.target_path))
|
||||
dump_basename = "{0}_{1}.sql".format(self.dbhost, self.dbname)
|
||||
|
||||
utils.execute('mv -f "{0}" "{1}"'.format(
|
||||
temp,
|
||||
os.path.join(self.target_path, dump_basename)))
|
||||
|
||||
|
||||
class XmlRpc(Driver):
|
||||
def __init__(self, data, conf):
|
||||
super(XmlRpc, self).__init__(data, conf)
|
||||
|
||||
self.server = self.data.get("server", "localhost")
|
||||
self.methods = self.data.get("methods", [])
|
||||
self.to_file = self.data.get("to_file")
|
||||
|
||||
self.target_path = os.path.join(
|
||||
self.conf.target, self.host, "xmlrpc", self.to_file)
|
||||
|
||||
def snapshot(self):
|
||||
utils.execute('mkdir -p "{0}"'.format(os.path.dirname(
|
||||
self.target_path)))
|
||||
|
||||
server = xmlrpclib.Server(self.server)
|
||||
with open(self.target_path, "w") as f:
|
||||
for method in self.methods:
|
||||
if hasattr(server, method):
|
||||
response = getattr(server, method)()
|
||||
response = pprint.pformat(response, indent=2)
|
||||
else:
|
||||
response = "no such method on remote server"
|
||||
|
||||
f.write("===== {0} =====\n{1}\n\n".format(method, response))
|
||||
|
||||
|
||||
class Command(Driver):
|
||||
|
||||
def __init__(self, data, conf):
|
||||
super(Command, self).__init__(data, conf)
|
||||
self.cmdname = self.data["command"]
|
||||
self.to_file = self.data["to_file"]
|
||||
self.target_path = os.path.join(
|
||||
self.conf.target, self.host, "commands", self.to_file)
|
||||
|
||||
def snapshot(self):
|
||||
out = self.command(self.cmdname)
|
||||
utils.execute('mkdir -p "{0}"'.format(os.path.dirname(
|
||||
self.target_path)))
|
||||
with open(self.target_path, "w") as f:
|
||||
f.write("===== COMMAND =====: {0}\n".format(self.cmdname))
|
||||
f.write("===== RETURN CODE =====: {0}\n".format(out.return_code))
|
||||
f.write("===== STDOUT =====:\n")
|
||||
if out.stdout:
|
||||
f.write(out.stdout)
|
||||
f.write("\n===== STDERR =====:\n")
|
||||
if out.stderr:
|
||||
f.write(out.stderr)
|
|
@ -1,37 +0,0 @@
|
|||
# Copyright 2013 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import logging
|
||||
|
||||
from shotgun.settings import LOG_FILE
|
||||
|
||||
|
||||
def configure_logger():
|
||||
"""Configures shotgun logger"""
|
||||
logger = logging.getLogger('shotgun')
|
||||
logger.setLevel(logging.DEBUG)
|
||||
formatter = logging.Formatter(
|
||||
'%(asctime)s %(levelname)s %(process)d (%(module)s) %(message)s',
|
||||
"%Y-%m-%d %H:%M:%S")
|
||||
|
||||
stream_handler = logging.StreamHandler()
|
||||
stream_handler.setLevel(logging.DEBUG)
|
||||
stream_handler.setFormatter(formatter)
|
||||
|
||||
file_handler = logging.FileHandler(LOG_FILE)
|
||||
file_handler.setLevel(logging.DEBUG)
|
||||
file_handler.setFormatter(formatter)
|
||||
|
||||
logger.addHandler(stream_handler)
|
||||
logger.addHandler(file_handler)
|
|
@ -1,43 +0,0 @@
|
|||
# Copyright 2013 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
from shotgun.driver import Driver
|
||||
from shotgun import utils
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Manager(object):
|
||||
def __init__(self, conf):
|
||||
logger.debug("Initializing snapshot manager")
|
||||
self.conf = conf
|
||||
|
||||
def snapshot(self):
|
||||
logger.debug("Making snapshot")
|
||||
utils.execute("rm -rf {0}".format(os.path.dirname(self.conf.target)))
|
||||
for obj_data in self.conf.objects:
|
||||
logger.debug("Dumping: %s", obj_data)
|
||||
driver = Driver.getDriver(obj_data, self.conf)
|
||||
driver.snapshot()
|
||||
logger.debug("Archiving dump directory: %s", self.conf.target)
|
||||
|
||||
utils.compress(self.conf.target, self.conf.compression_level)
|
||||
|
||||
with open(self.conf.lastdump, "w") as fo:
|
||||
fo.write("{0}.tar.xz".format(self.conf.target))
|
||||
return "{0}.tar.xz".format(self.conf.target)
|
|
@ -1,20 +0,0 @@
|
|||
# Copyright 2013 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
TARGET = "/tmp/snapshot"
|
||||
LASTDUMP = "/tmp/snapshot_last"
|
||||
TIMESTAMP = True
|
||||
COMPRESSION_LEVEL = 3
|
||||
LOG_FILE = "/var/log/shotgun.log"
|
||||
DEFAULT_TIMEOUT = 10
|
|
@ -1,13 +0,0 @@
|
|||
# Copyright 2013 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
|
@ -1,19 +0,0 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from unittest2.case import TestCase
|
||||
|
||||
|
||||
class BaseTestCase(TestCase):
|
||||
"""Base unit test case for shotgun tests."""
|
|
@ -1,58 +0,0 @@
|
|||
# Copyright 2013 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import time
|
||||
|
||||
import mock
|
||||
|
||||
from shotgun.config import Config
|
||||
from shotgun.test import base
|
||||
|
||||
|
||||
class TestConfig(base.BaseTestCase):
|
||||
|
||||
def test_timestamp(self):
|
||||
t = time.localtime()
|
||||
with mock.patch('shotgun.config.time') as MockedTime:
|
||||
MockedTime.localtime.return_value = t
|
||||
MockedTime.strftime.side_effect = time.strftime
|
||||
conf = Config({})
|
||||
stamped = conf._timestamp("sample")
|
||||
self.assertEqual(
|
||||
stamped,
|
||||
"sample-{0}".format(time.strftime('%Y-%m-%d_%H-%M-%S', t))
|
||||
)
|
||||
|
||||
def test_target_timestamp(self):
|
||||
conf = Config({
|
||||
"target": "/tmp/sample",
|
||||
"timestamp": True
|
||||
})
|
||||
self.assertRegex(
|
||||
conf.target,
|
||||
ur"\/tmp\/sample\-[\d]{4}\-[\d]{2}\-[\d]{2}_"
|
||||
"([\d]{2}\-){2}[\d]{2}",
|
||||
)
|
||||
|
||||
@mock.patch('shotgun.config.settings')
|
||||
def test_timeout(self, m_settings):
|
||||
conf = Config({})
|
||||
self.assertIs(conf.timeout, m_settings.DEFAULT_TIMEOUT)
|
||||
|
||||
def test_pass_default_timeout(self):
|
||||
timeout = 1345
|
||||
conf = Config({
|
||||
'timeout': timeout,
|
||||
})
|
||||
self.assertEqual(conf.timeout, timeout)
|
|
@ -1,219 +0,0 @@
|
|||
# Copyright 2013 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import random
|
||||
import sys
|
||||
|
||||
import fabric
|
||||
import mock
|
||||
|
||||
import shotgun
|
||||
from shotgun.test import base
|
||||
|
||||
|
||||
class RunOut(object):
|
||||
return_code = None
|
||||
stderr = None
|
||||
stdout = None
|
||||
|
||||
def __str__(self):
|
||||
return str(self.stdout)
|
||||
|
||||
|
||||
class TestDriver(base.BaseTestCase):
|
||||
def test_driver_factory(self):
|
||||
types = {
|
||||
"file": "File",
|
||||
"dir": "Dir",
|
||||
"postgres": "Postgres",
|
||||
"command": "Command"
|
||||
}
|
||||
for t, n in types.iteritems():
|
||||
with mock.patch("shotgun.driver.%s" % n) as mocked:
|
||||
shotgun.driver.Driver.getDriver({"type": t}, None)
|
||||
mocked.assert_called_with({"type": t}, None)
|
||||
|
||||
@mock.patch('shotgun.driver.utils.CCStringIO')
|
||||
@mock.patch('shotgun.driver.fabric.api.settings')
|
||||
@mock.patch('shotgun.driver.fabric.api.run')
|
||||
def test_driver_remote_command(self, mfabrun, mfabset, mccstring):
|
||||
out = shotgun.driver.CommandOut()
|
||||
out.stdout = "STDOUT"
|
||||
out.return_code = "RETURN_CODE"
|
||||
mccstring.return_value.getvalue.return_value = out.stdout
|
||||
|
||||
runout = RunOut()
|
||||
runout.return_code = "RETURN_CODE"
|
||||
mfabrun.return_value = runout
|
||||
|
||||
command = "COMMAND"
|
||||
|
||||
conf = mock.Mock()
|
||||
driver = shotgun.driver.Driver(
|
||||
{"host": {"address": "remote_host"}}, conf)
|
||||
result = driver.command(command)
|
||||
|
||||
mfabrun.assert_called_with(
|
||||
command, stdout=mock.ANY)
|
||||
mfabset.assert_called_with(
|
||||
host_string="remote_host",
|
||||
timeout=2,
|
||||
command_timeout=driver.timeout,
|
||||
warn_only=True,
|
||||
key_filename=None,
|
||||
abort_on_prompts=True)
|
||||
self.assertEqual(result, out)
|
||||
|
||||
@mock.patch('shotgun.driver.fabric.api.run')
|
||||
@mock.patch('shotgun.driver.fabric.api.settings')
|
||||
def test_fabric_use_timout_from_driver(self, mfabset, _):
|
||||
timeout = random.randint(1, 100)
|
||||
conf = mock.Mock()
|
||||
driver = shotgun.driver.Driver(
|
||||
{"host": {"address": "remote_host"}}, conf)
|
||||
driver.timeout = timeout
|
||||
driver.command("COMMAND")
|
||||
mfabset.assert_called_with(
|
||||
host_string=mock.ANY,
|
||||
timeout=mock.ANY,
|
||||
command_timeout=timeout,
|
||||
warn_only=mock.ANY,
|
||||
key_filename=mock.ANY,
|
||||
abort_on_prompts=mock.ANY)
|
||||
|
||||
@mock.patch('shotgun.driver.utils.execute')
|
||||
def test_driver_local_command(self, mexecute):
|
||||
mexecute.return_value = ("RETURN_CODE", "STDOUT", "STDERR")
|
||||
|
||||
out = shotgun.driver.CommandOut()
|
||||
out.stdout = "STDOUT"
|
||||
out.stderr = "STDERR"
|
||||
out.return_code = "RETURN_CODE"
|
||||
|
||||
command = "COMMAND"
|
||||
conf = mock.Mock()
|
||||
driver = shotgun.driver.Driver({}, conf)
|
||||
result = driver.command(command)
|
||||
shotgun.driver.utils.execute.assert_called_with(command)
|
||||
self.assertEqual(result, out)
|
||||
|
||||
@mock.patch('shotgun.driver.utils.CCStringIO')
|
||||
@mock.patch('shotgun.driver.fabric.api.settings')
|
||||
@mock.patch('shotgun.driver.fabric.api.run')
|
||||
def test_command_timeout(self, mfabrun, mfabset, mstringio):
|
||||
mfabrun.side_effect = fabric.exceptions.CommandTimeout(10)
|
||||
|
||||
mstdout = mock.MagicMock()
|
||||
mstdout.getvalue.return_value = 'FULL STDOUT'
|
||||
mstringio.return_value = mstdout
|
||||
|
||||
command = "COMMAND"
|
||||
|
||||
conf = mock.Mock()
|
||||
driver = shotgun.driver.Driver(
|
||||
{"host": {"address": "remote_host"}}, conf)
|
||||
result = driver.command(command)
|
||||
|
||||
mstringio.assert_has_calls([
|
||||
mock.call(writers=sys.stdout),
|
||||
])
|
||||
mfabrun.assert_called_with(command, stdout=mstdout)
|
||||
self.assertEqual(result.stdout, 'FULL STDOUT')
|
||||
|
||||
@mock.patch('shotgun.driver.utils.execute')
|
||||
@mock.patch('shotgun.driver.fabric.api.settings')
|
||||
@mock.patch('shotgun.driver.fabric.api.get')
|
||||
def test_driver_get(self, mfabget, mfabset, mexecute):
|
||||
mexecute.return_value = ("RETURN_CODE", "STDOUT", "STDERR")
|
||||
remote_path = "/remote_dir/remote_file"
|
||||
target_path = "/target_dir"
|
||||
conf = mock.Mock()
|
||||
|
||||
driver = shotgun.driver.Driver({
|
||||
"host": {
|
||||
"address": "remote_host",
|
||||
"ssh-key": "path_to_key",
|
||||
}
|
||||
}, conf)
|
||||
driver.get(remote_path, target_path)
|
||||
mexecute.assert_called_with('mkdir -p "{0}"'.format(target_path))
|
||||
mfabget.assert_called_with(remote_path, target_path)
|
||||
mfabset.assert_called_with(
|
||||
host_string="remote_host", key_filename="path_to_key",
|
||||
timeout=2, warn_only=True, abort_on_prompts=True)
|
||||
|
||||
mexecute.reset_mock()
|
||||
driver = shotgun.driver.Driver({}, conf)
|
||||
driver.get(remote_path, target_path)
|
||||
self.assertEqual(mexecute.mock_calls, [
|
||||
mock.call('mkdir -p "{0}"'.format(target_path)),
|
||||
mock.call('cp -r "{0}" "{1}"'.format(remote_path, target_path))])
|
||||
|
||||
def test_use_timeout_from_global_conf(self):
|
||||
data = {}
|
||||
conf = mock.Mock(spec=shotgun.config.Config, target="some_target")
|
||||
cmd_driver = shotgun.driver.Driver(data, conf)
|
||||
self.assertEqual(cmd_driver.timeout, conf.timeout)
|
||||
|
||||
def test_use_command_specific_timeout(self):
|
||||
timeout = 1234
|
||||
data = {
|
||||
"timeout": timeout
|
||||
}
|
||||
conf = mock.Mock(spec=shotgun.config.Config, target="some_target")
|
||||
cmd_driver = shotgun.driver.Driver(data, conf)
|
||||
self.assertEqual(cmd_driver.timeout, timeout)
|
||||
self.assertNotEqual(cmd_driver.timeout, conf.timeout)
|
||||
|
||||
|
||||
class TestFile(base.BaseTestCase):
|
||||
|
||||
@mock.patch('shotgun.driver.Driver.get')
|
||||
def test_snapshot(self, mget):
|
||||
data = {
|
||||
"type": "file",
|
||||
"path": "/remote_dir/remote_file",
|
||||
"host": {
|
||||
"address": "remote_host",
|
||||
},
|
||||
}
|
||||
conf = mock.MagicMock()
|
||||
conf.target = "/target"
|
||||
file_driver = shotgun.driver.File(data, conf)
|
||||
|
||||
target_path = "/target/remote_host/remote_dir"
|
||||
file_driver.snapshot()
|
||||
|
||||
mget.assert_called_with(data["path"], target_path)
|
||||
|
||||
@mock.patch('shotgun.driver.utils.remove')
|
||||
@mock.patch('shotgun.driver.Driver.get')
|
||||
def test_dir_exclude_called(self, mget, mremove):
|
||||
data = {
|
||||
"type": "dir",
|
||||
"path": "/remote_dir/",
|
||||
"exclude": ["*test"],
|
||||
"host": {
|
||||
"address": "remote_host",
|
||||
},
|
||||
}
|
||||
conf = mock.MagicMock()
|
||||
conf.target = "/target"
|
||||
dir_driver = shotgun.driver.Dir(data, conf)
|
||||
|
||||
target_path = "/target/remote_host/remote_dir"
|
||||
dir_driver.snapshot()
|
||||
|
||||
mget.assert_called_with(data["path"], target_path)
|
||||
mremove.assert_called_with(dir_driver.full_dst_path, data['exclude'])
|
|
@ -1,43 +0,0 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import tempfile
|
||||
|
||||
import mock
|
||||
|
||||
from shotgun.manager import Manager
|
||||
from shotgun.test import base
|
||||
|
||||
|
||||
class TestManager(base.BaseTestCase):
|
||||
|
||||
@mock.patch('shotgun.manager.Driver.getDriver')
|
||||
@mock.patch('shotgun.manager.utils.execute')
|
||||
@mock.patch('shotgun.manager.utils.compress')
|
||||
def test_snapshot(self, mcompress, mexecute, mget):
|
||||
data = {
|
||||
"type": "file",
|
||||
"path": "/remote_dir/remote_file",
|
||||
"host": {
|
||||
"address": "remote_host",
|
||||
},
|
||||
}
|
||||
conf = mock.MagicMock()
|
||||
conf.target = "/target/data"
|
||||
conf.objects = [data]
|
||||
conf.lastdump = tempfile.mkstemp()[1]
|
||||
manager = Manager(conf)
|
||||
manager.snapshot()
|
||||
mget.assert_called_once_with(data, conf)
|
||||
mexecute.assert_called_once_with('rm -rf /target')
|
|
@ -1,112 +0,0 @@
|
|||
# coding: utf-8
|
||||
|
||||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import StringIO
|
||||
|
||||
import mock
|
||||
|
||||
from shotgun.test import base
|
||||
from shotgun import utils
|
||||
|
||||
|
||||
class TestUtils(base.BaseTestCase):
|
||||
|
||||
@mock.patch('shotgun.utils.execute')
|
||||
def test_remove_subdir(self, mexecute):
|
||||
utils.remove('/', ['good', '**/*.py'])
|
||||
mexecute.assert_has_calls([
|
||||
mock.call('shopt -s globstar; rm -rf /good', shell=True),
|
||||
mock.call('shopt -s globstar; rm -rf /**/*.py', shell=True)])
|
||||
|
||||
@mock.patch('shotgun.utils.os.walk')
|
||||
def test_iterfiles(self, mwalk):
|
||||
path = '/root'
|
||||
mwalk.return_value = [
|
||||
(path, '', ('file1', 'file2')),
|
||||
(path + '/sub', '', ('file3',))]
|
||||
|
||||
result = list(utils.iterfiles(path))
|
||||
|
||||
mwalk.assert_called_once_with(path, topdown=True)
|
||||
self.assertEqual(
|
||||
result, ['/root/file1', '/root/file2', '/root/sub/file3'])
|
||||
|
||||
@mock.patch('shotgun.utils.execute')
|
||||
def test_compress(self, mexecute):
|
||||
target = '/path/target'
|
||||
level = '-3'
|
||||
|
||||
utils.compress(target, level)
|
||||
|
||||
compress_call = mexecute.call_args_list[0]
|
||||
rm_call = mexecute.call_args_list[1]
|
||||
|
||||
compress_env = compress_call[1]['env']
|
||||
self.assertEqual(compress_env['XZ_OPT'], level)
|
||||
self.assertEqual(
|
||||
compress_call[0][0],
|
||||
'tar cJvf /path/target.tar.xz -C /path target')
|
||||
|
||||
self.assertEqual(rm_call[0][0], 'rm -r /path/target')
|
||||
|
||||
|
||||
class TestCCStringIO(base.BaseTestCase):
|
||||
|
||||
def test_no_writers(self):
|
||||
test_string = 'some_string'
|
||||
|
||||
ccstring = utils.CCStringIO()
|
||||
ccstring.write(test_string)
|
||||
|
||||
self.assertEqual(ccstring.getvalue(), test_string)
|
||||
|
||||
def test_with_one_writer(self):
|
||||
test_string = 'some_string'
|
||||
|
||||
writer = StringIO.StringIO()
|
||||
ccstring = utils.CCStringIO(writers=writer)
|
||||
ccstring.write(test_string)
|
||||
|
||||
self.assertEqual(ccstring.getvalue(), test_string)
|
||||
self.assertEqual(writer.getvalue(), test_string)
|
||||
|
||||
def test_with_multiple_writers(self):
|
||||
test_string = 'some_string'
|
||||
|
||||
writer_a = StringIO.StringIO()
|
||||
writer_b = StringIO.StringIO()
|
||||
ccstring = utils.CCStringIO(writers=[writer_a, writer_b])
|
||||
ccstring.write(test_string)
|
||||
|
||||
self.assertEqual(ccstring.getvalue(), test_string)
|
||||
self.assertEqual(writer_a.getvalue(), test_string)
|
||||
self.assertEqual(writer_b.getvalue(), test_string)
|
||||
|
||||
def test_with_writer_and_buffer(self):
|
||||
buffer = 'I am here already'
|
||||
|
||||
writer = StringIO.StringIO()
|
||||
ccstring = utils.CCStringIO(buffer, writers=writer)
|
||||
|
||||
self.assertEqual(ccstring.getvalue(), buffer)
|
||||
self.assertEqual(writer.getvalue(), '')
|
||||
|
||||
def test_non_ascii_output_with_unicode(self):
|
||||
ccstring = utils.CCStringIO()
|
||||
ccstring.write('привет')
|
||||
ccstring.write(u'test')
|
||||
|
||||
self.assertEqual(ccstring.getvalue(), 'приветtest')
|
|
@ -1,158 +0,0 @@
|
|||
# Copyright 2013 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import copy
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import socket
|
||||
from StringIO import StringIO
|
||||
import subprocess
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def hostname():
|
||||
return socket.gethostname()
|
||||
|
||||
|
||||
def is_ip(name):
|
||||
return (re.search(ur"([0-9]{1,3}\.){3}[0-9]{1,3}", name) and True)
|
||||
|
||||
|
||||
def fqdn(name=None):
|
||||
if name:
|
||||
return socket.getfqdn(name)
|
||||
return socket.getfqdn(socket.gethostname())
|
||||
|
||||
|
||||
def is_local(name):
|
||||
if name in ("localhost", hostname(), fqdn()):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def iterfiles(path):
|
||||
for root, dirnames, filenames in os.walk(path, topdown=True):
|
||||
for filename in filenames:
|
||||
yield os.path.join(root, filename)
|
||||
|
||||
|
||||
def remove(full_dst_path, excludes):
|
||||
"""Removes subdirs/files using unixs syntax
|
||||
|
||||
full_dst_path is treated as root directory for remove
|
||||
|
||||
:param full_dst_path: str
|
||||
:param excludes: list with excludes paths/files
|
||||
"""
|
||||
for exclude in excludes:
|
||||
path = os.path.join(full_dst_path, exclude.lstrip('/'))
|
||||
logger.debug('Deleting %s', path)
|
||||
execute("shopt -s globstar; rm -rf {0}".format(path), shell=True)
|
||||
|
||||
|
||||
def compress(target, level, keep_target=False):
|
||||
"""Runs compression of provided directory
|
||||
|
||||
:param target: directory to compress
|
||||
:param level: level of compression
|
||||
:param keep_target: bool, if True target directory wont be removed
|
||||
"""
|
||||
env = copy.deepcopy(os.environ)
|
||||
env['XZ_OPT'] = level
|
||||
execute("tar cJvf {0}.tar.xz -C {1} {2}"
|
||||
"".format(target,
|
||||
os.path.dirname(target),
|
||||
os.path.basename(target)),
|
||||
env=env)
|
||||
if not keep_target:
|
||||
execute("rm -r {0}".format(target))
|
||||
|
||||
|
||||
def execute(command, to_filename=None, env=None, shell=False):
|
||||
logger.debug("Trying to execute command: %s", command)
|
||||
commands = [c.strip() for c in re.split(ur'\|', command)]
|
||||
env = env or os.environ
|
||||
env["PATH"] = "/bin:/usr/bin:/sbin:/usr/sbin"
|
||||
|
||||
to_file = None
|
||||
if to_filename:
|
||||
to_file = open(to_filename, 'wb')
|
||||
|
||||
process = []
|
||||
for c in commands:
|
||||
try:
|
||||
# NOTE(eli): Python's shlex implementation doesn't like unicode.
|
||||
# We have to convert to ascii before shlex'ing the command.
|
||||
# http://bugs.python.org/issue6988
|
||||
encoded_command = c.encode('ascii')
|
||||
process.append(subprocess.Popen(
|
||||
shlex.split(encoded_command) if not shell else encoded_command,
|
||||
env=env,
|
||||
stdin=(process[-1].stdout if process else None),
|
||||
stdout=(to_file
|
||||
if (len(process) == len(commands) - 1) and to_file
|
||||
else subprocess.PIPE),
|
||||
stderr=(subprocess.PIPE),
|
||||
shell=shell
|
||||
))
|
||||
except OSError as e:
|
||||
return (1, "", "{0}\n".format(e))
|
||||
|
||||
if len(process) >= 2:
|
||||
process[-2].stdout.close()
|
||||
stdout, stderr = process[-1].communicate()
|
||||
return (process[-1].returncode, stdout, stderr)
|
||||
|
||||
|
||||
class CCStringIO(StringIO):
|
||||
"""A "carbon copy" StringIO.
|
||||
|
||||
It's capable of multiplexing its writes to other buffer objects.
|
||||
|
||||
Taken from fabric.tests.mock_streams.CarbonCopy
|
||||
"""
|
||||
|
||||
def __init__(self, buffer='', writers=None):
|
||||
"""CCStringIO initializator
|
||||
|
||||
If ``writers`` is given and is a file-like object or an
|
||||
iterable of same, it/they will be written to whenever this
|
||||
StringIO instance is written to.
|
||||
"""
|
||||
StringIO.__init__(self, buffer)
|
||||
if writers is None:
|
||||
writers = []
|
||||
elif hasattr(writers, 'write'):
|
||||
writers = [writers]
|
||||
self.writers = writers
|
||||
|
||||
def write(self, s):
|
||||
# unfortunately, fabric writes into StringIO both so-called
|
||||
# bytestrings and unicode strings. obviously, bytestrings may
|
||||
# contain non-ascii symbols. that leads to type-conversion
|
||||
# issue when we use string's join (inside getvalue()) with
|
||||
# a list of both unicodes and bytestrings. in order to avoid
|
||||
# this issue we should convert all input unicode strings into
|
||||
# utf-8 bytestrings (let's assume that slaves encoding is utf-8
|
||||
# too so we won't have encoding mess in the output file).
|
||||
if isinstance(s, unicode):
|
||||
s = s.encode('utf-8')
|
||||
|
||||
StringIO.write(self, s)
|
||||
for writer in self.writers:
|
||||
writer.write(s)
|
|
@ -1,40 +0,0 @@
|
|||
%define name shotgun
|
||||
%{!?version: %define version 8.0.0}
|
||||
%{!?release: %define release 1}
|
||||
|
||||
Name: %{name}
|
||||
Summary: Shotgun package
|
||||
Version: %{version}
|
||||
Release: %{release}
|
||||
URL: http://mirantis.com
|
||||
License: Apache
|
||||
Group: Development/Libraries
|
||||
BuildRoot: %{_tmppath}/%{name}-%{version}-buildroot
|
||||
Prefix: %{_prefix}
|
||||
BuildArch: noarch
|
||||
Requires: postgresql
|
||||
Requires: python-fabric >= 1.10.0
|
||||
Requires: python-argparse
|
||||
Requires: tar
|
||||
Requires: gzip
|
||||
Requires: bzip2
|
||||
Requires: openssh-clients
|
||||
Requires: xz
|
||||
|
||||
%description
|
||||
Shotgun package.
|
||||
|
||||
%prep
|
||||
%setup -cq -n %{name}-%{version}
|
||||
|
||||
%build
|
||||
cd %{_builddir}/%{name}-%{version}/shotgun && python setup.py build
|
||||
|
||||
%install
|
||||
cd %{_builddir}/%{name}-%{version}/shotgun && python setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=%{_builddir}/%{name}-%{version}/shotgun/INSTALLED_FILES
|
||||
|
||||
%clean
|
||||
rm -rf $RPM_BUILD_ROOT
|
||||
|
||||
%files -f %{_builddir}/%{name}-%{version}/shotgun/INSTALLED_FILES
|
||||
%defattr(-,root,root)
|
|
@ -1,4 +0,0 @@
|
|||
mock==1.3.0
|
||||
unittest2==1.1.0
|
||||
pytest==2.8.0
|
||||
pytest-cov==2.1.0
|
|
@ -1,43 +0,0 @@
|
|||
[tox]
|
||||
minversion = 1.6
|
||||
skipsdist = True
|
||||
envlist = py26,py27,pep8
|
||||
|
||||
[testenv]
|
||||
usedevelop = True
|
||||
install_command = pip install --allow-external -U {opts} {packages}
|
||||
setenv = VIRTUAL_ENV={envdir}
|
||||
deps = -r{toxinidir}/test-requirements.txt
|
||||
commands =
|
||||
py.test -vv {posargs:shotgun/test}
|
||||
|
||||
[tox:jenkins]
|
||||
downloadcache = ~/cache/pip
|
||||
|
||||
[testenv:pep8]
|
||||
deps = hacking==0.10
|
||||
usedevelop = False
|
||||
commands =
|
||||
flake8 {posargs:shotgun}
|
||||
|
||||
[testenv:cover]
|
||||
setenv = VIRTUAL_ENV={envdir}
|
||||
commands =
|
||||
py.test -vv --cov=shotgun {posargs:shotgun/test}
|
||||
|
||||
[testenv:venv]
|
||||
commands = {posargs:}
|
||||
|
||||
[testenv:devenv]
|
||||
envdir = devenv
|
||||
usedevelop = True
|
||||
|
||||
[flake8]
|
||||
ignore = H234,H302,H802
|
||||
exclude = .venv,.git,.tox,dist,doc,*lib/python*,*egg,build,tools,__init__.py,docs
|
||||
show-pep8 = True
|
||||
show-source = True
|
||||
count = True
|
||||
|
||||
[hacking]
|
||||
import_exceptions = testtools.matchers
|
|
@ -117,13 +117,11 @@ cd %{_builddir}/%{name}-%{version}/nailgun && %{_builddir}/%{name}-%{version}/na
|
|||
mv %{_builddir}/%{name}-%{version}/nailgun/compressed_static %{_builddir}/%{name}-%{version}/nailgun/static
|
||||
cd %{_builddir}/%{name}-%{version}/nailgun && python setup.py build
|
||||
cd %{_builddir}/%{name}-%{version}/network_checker && python setup.py build
|
||||
cd %{_builddir}/%{name}-%{version}/shotgun && python setup.py build
|
||||
cd %{_builddir}/%{name}-%{version}/fuel_upgrade_system/fuel_package_updates && python setup.py build
|
||||
|
||||
%install
|
||||
cd %{_builddir}/%{name}-%{version}/nailgun && python setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=%{_builddir}/%{name}-%{version}/nailgun/INSTALLED_FILES
|
||||
cd %{_builddir}/%{name}-%{version}/network_checker && python setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=%{_builddir}/%{name}-%{version}/network_checker/INSTALLED_FILES
|
||||
cd %{_builddir}/%{name}-%{version}/shotgun && python setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=%{_builddir}/%{name}-%{version}/shotgun/INSTALLED_FILES
|
||||
cd %{_builddir}/%{name}-%{version}/fuel_upgrade_system/fuel_package_updates && python setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=%{_builddir}/%{name}-%{version}/fuel_upgrade_system/fuel_package_updates/INSTALLED_FILES
|
||||
mkdir -p %{buildroot}/opt/nailgun/bin
|
||||
mkdir -p %{buildroot}/etc/cron.d
|
||||
|
@ -185,31 +183,6 @@ between hosts in network.
|
|||
%files -n nailgun-net-check -f %{_builddir}/%{name}-%{version}/network_checker/INSTALLED_FILES
|
||||
%defattr(-,root,root)
|
||||
|
||||
%package -n shotgun
|
||||
|
||||
Summary: Shotgun package
|
||||
Version: %{version}
|
||||
Release: %{release}
|
||||
URL: http://mirantis.com
|
||||
License: Apache
|
||||
Group: Development/Libraries
|
||||
BuildRoot: %{_tmppath}/%{name}-%{version}-buildroot
|
||||
Prefix: %{_prefix}
|
||||
BuildArch: noarch
|
||||
Requires: postgresql
|
||||
Requires: python-fabric >= 1.10.0
|
||||
Requires: python-argparse
|
||||
Requires: tar
|
||||
Requires: gzip
|
||||
Requires: bzip2
|
||||
Requires: openssh-clients
|
||||
Requires: xz
|
||||
|
||||
%description -n shotgun
|
||||
Shotgun package.
|
||||
|
||||
%files -n shotgun -f %{_builddir}/%{name}-%{version}/shotgun/INSTALLED_FILES
|
||||
%defattr(-,root,root)
|
||||
|
||||
%package -n fencing-agent
|
||||
Summary: Fencing agent
|
||||
|
|
Loading…
Reference in New Issue