Deprecate shotgun directory

Change-Id: Ib1260df78a45e9cc8bf6bdfd9627508ce50ae812
Related-Bug: #1506894
This commit is contained in:
Vladimir Kozhukalov 2015-10-26 14:38:18 +03:00
parent 820a096c3a
commit 65b07097b2
30 changed files with 0 additions and 1433 deletions

View File

@ -49,15 +49,6 @@ maintainers:
email: loles@mirantis.com
IRC: salmon_
- shotgun/:
- name: Maciej Kwiek
email: mkwiek@mirantis.com
IRC: mkwiek
- name: Vladimir Kozhukalov
email: vkozhukalov@mirantis.com
IRC: kozhukalov
- nailgun/:
- name: Aleksandr Kislitskii
email: akislitsky@mirantis.com

View File

@ -88,7 +88,6 @@ prepare_venv() {
virtualenv "${VENV}" # you can use any name instead of 'fuel'
. "${VENV}/bin/activate" # command selects the particular environment
# install dependencies
pip install ./shotgun # this fuel project is listed in setup.py requirements
pip install -r 'nailgun/test-requirements.txt'
}

View File

@ -273,7 +273,6 @@ Now you can create the virtual environment and activate it.
And then install the dependencies.
::
pip install ./shotgun
pip install -r nailgun/test-requirements.txt
Now you can look at the list of available formats and generate

View File

@ -78,7 +78,6 @@ Preparing Development Environment
sudo apt-get install --yes git
git clone https://github.com/openstack/fuel-web.git
cd fuel-web
pip install ./shotgun # this fuel project is listed in setup.py requirements
pip install --allow-all-external -r nailgun/test-requirements.txt
#. Create required folder for log files::
@ -302,4 +301,3 @@ you could only run dropdb with *./run_tests.sh* script.
Now you need to run dropdb for each slave node:
the *py.test --cleandb <path to the tests>* command is introduced for this
purpose.

View File

@ -26,8 +26,6 @@ function usage {
echo " -x, --performance Run NAILGUN performance tests"
echo " -p, --flake8 Run FLAKE8 and HACKING compliance check"
echo " -P, --no-flake8 Don't run static code checks"
echo " -s, --shotgun Run SHOTGUN tests"
echo " -S, --no-shotgun Don't run SHOTGUN tests"
echo " -t, --tests Run a given test files"
echo " -u, --upgrade Run tests for UPGRADE system"
echo " -U, --no-upgrade Don't run tests for UPGRADE system"
@ -59,8 +57,6 @@ function process_options {
-x|--performance) performance_tests=1;;
-u|--upgrade) upgrade_system=1;;
-U|--no-upgrade) no_upgrade_system=1;;
-s|--shotgun) shotgun_tests=1;;
-S|--no-shotgun) no_shotgun_tests=1;;
-p|--flake8) flake8_checks=1;;
-P|--no-flake8) no_flake8_checks=1;;
-w|--webui) ui_lint_checks=1; ui_unit_tests=1; ui_func_tests=1;;
@ -100,7 +96,6 @@ testropts="--with-timer --timer-warning=10 --timer-ok=2 --timer-top-n=10"
# nosetest xunit options
NAILGUN_XUNIT=${NAILGUN_XUNIT:-"$ROOT/nailgun.xml"}
FUELUPGRADE_XUNIT=${FUELUPGRADE_XUNIT:-"$ROOT/fuelupgrade.xml"}
SHOTGUN_XUNIT=${SHOTGUN_XUNIT:-"$ROOT/shotgun.xml"}
EXTENSIONS_XUNIT=${EXTENSIONS_XUNIT:-"$ROOT/extensions.xml"}
NAILGUN_PORT=${NAILGUN_PORT:-5544}
TEST_NAILGUN_DB=${TEST_NAILGUN_DB:-nailgun}
@ -118,8 +113,6 @@ no_nailgun_tests=0
performance_tests=0
upgrade_system=0
no_upgrade_system=0
shotgun_tests=0
no_shotgun_tests=0
flake8_checks=0
no_flake8_checks=0
ui_lint_checks=0
@ -162,7 +155,6 @@ function run_tests {
$ui_unit_tests -eq 0 && \
$ui_func_tests -eq 0 && \
$upgrade_system -eq 0 && \
$shotgun_tests -eq 0 && \
$extensions_tests -eq 0 && \
$flake8_checks -eq 0 ]]; then
@ -171,7 +163,6 @@ function run_tests {
if [ $no_ui_unit_tests -ne 1 ]; then ui_unit_tests=1; fi
if [ $no_ui_func_tests -ne 1 ]; then ui_func_tests=1; fi
if [ $no_upgrade_system -ne 1 ]; then upgrade_system=1; fi
if [ $no_shotgun_tests -ne 1 ]; then shotgun_tests=1; fi
if [ $no_flake8_checks -ne 1 ]; then flake8_checks=1; fi
if [ $no_extensions_tests -ne 1 ]; then extensions_tests=1; fi
@ -208,11 +199,6 @@ function run_tests {
run_upgrade_system_tests || errors+=" upgrade_system_tests"
fi
if [ $shotgun_tests -eq 1 ]; then
echo "Starting Shotgun tests..."
run_shotgun_tests || errors+=" shotgun_tests"
fi
if [ $extensions_tests -eq 1 ]; then
echo "Starting Extensions tests..."
run_extensions_tests || errors+=" extensions_tests"
@ -377,25 +363,6 @@ function run_upgrade_system_tests {
}
# Run shotgun tests
#
# Arguments:
#
# $@ -- tests to be run; with no arguments all tests will be run
function run_shotgun_tests {
local result=0
pushd $ROOT/shotgun >> /dev/null
# run tests
TOXENV=$TOXENV \
tox || result=1
popd >> /dev/null
return $result
}
function run_flake8_subproject {
local DIRECTORY=$1
local result=0
@ -435,7 +402,6 @@ function run_flake8 {
run_flake8_subproject network_checker && \
run_flake8_subproject fuel_upgrade_system/fuel_upgrade && \
run_flake8_subproject fuel_upgrade_system/fuel_package_updates && \
run_flake8_subproject shotgun || result=1
return $result
}
@ -619,8 +585,6 @@ function guess_test_run {
run_ui_func_tests $1
elif [[ $1 == *fuel_upgrade_system* ]]; then
run_upgrade_system_tests $1
elif [[ $1 == *shotgun* ]]; then
run_shotgun_tests $1
else
run_nailgun_tests $1
fi

View File

@ -1,3 +0,0 @@
[run]
branch = True
omit = shotgun/test/*

View File

@ -1,4 +0,0 @@
[DEFAULT]
test_command=OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 OS_TEST_TIMEOUT=60 ${PYTHON:-python} -m subunit.run discover -s shotgun/test/ -p "*.py" $LISTOPT $IDOPTION
test_id_option=--load-list $IDFILE
test_list_option=--list

View File

@ -1,40 +0,0 @@
{
"target": "/tmp/snapshot",
"timestamp": "true",
"dump_roles": {
"master": ["localhost"],
"slave": ["srv11-msk.msk.mirantis.net"]
},
"dump_objects": {
"master": [
{
"type": "file",
"path": "/etc/rsyslog.d"
},
{
"type": "postgres",
"dbname": "nailgun",
"username": "postgres"
},
],
"slave": [
{
"type": "command",
"command": "ps auxww"
},
{
"type": "command",
"command": "top"
},
{
"type": "command",
"command": "ip a"
},
{
"type": "file",
"path": "/etc/naily.facts"
}
]
}
}

View File

@ -1,34 +0,0 @@
# Copyright 2013 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import json
import logging
import os
import sys
sys.path[:0] = [os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))]
from shotgun.config import Config
from shotgun.manager import Manager
logging.basicConfig(level=logging.DEBUG)
with open("snapshot.json", "r") as fo:
data = json.loads(fo.read())
config = Config(data)
manager = Manager(config)
manager.snapshot()

View File

@ -1 +0,0 @@
Fabric>=1.7.0

View File

@ -1,20 +0,0 @@
#!/bin/bash
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -e
set -x
tox -v

View File

@ -1,36 +0,0 @@
# Copyright 2013 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import setuptools
setuptools.setup(
name='shotgun',
version='8.0.0',
description='Shotgun package',
long_description='Shotgun is diagnostic snapshot generator',
classifiers=[
'Development Status :: 4 - Beta',
'Programming Language :: Python'],
author='Mirantis Inc.',
author_email='product@mirantis.com',
url='http://mirantis.com',
keywords='shotgun mirantis',
packages=setuptools.find_packages(),
zip_safe=False,
install_requires=[
'Fabric >= 1.10.0'],
entry_points={
'console_scripts': [
'shotgun = shotgun.cli:main']})

View File

@ -1,13 +0,0 @@
# Copyright 2013 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

View File

@ -1,71 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import argparse
import json
import logging
from shotgun.logger import configure_logger
configure_logger()
from shotgun.config import Config
from shotgun.manager import Manager
logger = logging.getLogger(__name__)
def parse_args():
"""Parse arguments and return them
:returns: argparse object
"""
parser = argparse.ArgumentParser()
parser.add_argument(
'-c',
'--config',
help='configuration file',
required=True)
return parser.parse_args()
def read_config(config_path):
"""Reads config
:param config_path: path to configuration file
:returns: dict with configuration data
"""
with open(config_path, "r") as fo:
config = json.loads(fo.read())
return config
def make_snapshot(args):
"""Generates snapshot
:param args: argparse object
"""
config_object = Config(read_config(args.config))
manager = Manager(config_object)
snapshot_path = manager.snapshot()
logger.info(u'Snapshot path: {0}'.format(snapshot_path))
def main():
"""Entry point"""
make_snapshot(parse_args())

View File

@ -1,69 +0,0 @@
# Copyright 2013 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import logging
import time
from shotgun import settings
logger = logging.getLogger(__name__)
class Config(object):
def __init__(self, data=None):
self.data = data
self.time = time.localtime()
def _timestamp(self, name):
return "{0}-{1}".format(
name,
time.strftime('%Y-%m-%d_%H-%M-%S', self.time)
)
@property
def target(self):
target = self.data.get("target", settings.TARGET)
if self.data.get("timestamp", settings.TIMESTAMP):
target = self._timestamp(target)
return target
@property
def compression_level(self):
level = self.data.get("compression_level")
if level is None:
logger.info(
'Compression level is not specified,'
' Default %s will be used', settings.COMPRESSION_LEVEL)
level = settings.COMPRESSION_LEVEL
return '-{level}'.format(level=level)
@property
def lastdump(self):
return self.data.get("lastdump", settings.LASTDUMP)
@property
def objects(self):
for role, properties in self.data["dump"].iteritems():
for host in properties.get("hosts", []):
for object_ in properties.get("objects", []):
object_["host"] = host
yield object_
@property
def timeout(self):
"""Timeout for executing commands."""
return self.data.get("timeout", settings.DEFAULT_TIMEOUT)

View File

@ -1,257 +0,0 @@
# Copyright 2013 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import logging
import os
import pprint
import pwd
import re
import stat
import sys
import xmlrpclib
import fabric.api
from shotgun import utils
logger = logging.getLogger(__name__)
class CommandOut(object):
stdout = None
return_code = None
stderr = None
def __eq__(self, other):
return (
str(self.stdout) == str(other.stdout) and
str(self.stderr) == str(other.stderr) and
str(self.return_code) == str(other.return_code)
)
class Driver(object):
@classmethod
def getDriver(cls, data, conf):
driver_type = data["type"]
return {
"file": File,
"dir": Dir,
"postgres": Postgres,
"xmlrpc": XmlRpc,
"command": Command,
}.get(driver_type, cls)(data, conf)
def __init__(self, data, conf):
logger.debug("Initializing driver %s: host=%s",
self.__class__.__name__, data.get("host"))
self.data = data
self.host = self.data.get("host", {}).get("address", "localhost")
self.ssh_key = self.data.get("host", {}).get("ssh-key")
self.local = utils.is_local(self.host)
self.conf = conf
self.timeout = self.data.get("timeout", self.conf.timeout)
def snapshot(self):
raise NotImplementedError
def command(self, command):
out = CommandOut()
raw_stdout = utils.CCStringIO(writers=sys.stdout)
try:
if not self.local:
with fabric.api.settings(
host_string=self.host, # destination host
key_filename=self.ssh_key, # a path to ssh key
timeout=2, # a network connection timeout
command_timeout=self.timeout, # command execution timeout
warn_only=True, # don't exit on error
abort_on_prompts=True, # non-interactive mode
):
logger.debug("Running remote command: "
"host: %s command: %s", self.host, command)
try:
output = fabric.api.run(command, stdout=raw_stdout)
except SystemExit:
logger.error("Fabric aborted this iteration")
# NOTE(prmtl): because of pty=True (default) and
# combine_stderr=True (default) stderr is combined
# with stdout
out.stdout = raw_stdout.getvalue()
out.return_code = output.return_code
else:
logger.debug("Running local command: %s", command)
out.return_code, out.stdout, out.stderr = utils.execute(
command)
except Exception as e:
logger.error("Error occured: %s", str(e))
out.stdout = raw_stdout.getvalue()
return out
def get(self, path, target_path):
"""Get remote or local file
target_path must be the directory where to put
copied files or directories
"""
try:
if not self.local:
with fabric.api.settings(
host_string=self.host, # destination host
key_filename=self.ssh_key, # a path to ssh key
timeout=2, # a network connection timeout
warn_only=True, # don't exit on error
abort_on_prompts=True, # non-interactive mode
):
logger.debug("Getting remote file: %s %s",
path, target_path)
utils.execute('mkdir -p "{0}"'.format(target_path))
try:
return fabric.api.get(path, target_path)
except SystemExit:
logger.error("Fabric aborted this iteration")
else:
logger.debug("Getting local file: cp -r %s %s",
path, target_path)
utils.execute('mkdir -p "{0}"'.format(target_path))
return utils.execute('cp -r "{0}" "{1}"'.format(path,
target_path))
except Exception as e:
logger.error("Error occured: %s", str(e))
class File(Driver):
def __init__(self, data, conf):
super(File, self).__init__(data, conf)
self.path = self.data["path"]
self.exclude = self.data.get('exclude', [])
logger.debug("File to get: %s", self.path)
self.target_path = str(os.path.join(
self.conf.target, self.host,
os.path.dirname(self.path).lstrip("/")))
self.full_dst_path = os.path.join(
self.conf.target, self.host,
self.path.lstrip("/"))
logger.debug("File to save: %s", self.target_path)
def snapshot(self):
"""Make a snapshot
Example:
self.conf.target IS /target
self.host IS host.domain.tld
self.path IS /var/log/somedir
self.target_path IS /target/host.domain.tld/var/log
"""
self.get(self.path, self.target_path)
if self.exclude:
utils.remove(self.full_dst_path, self.exclude)
Dir = File
class Postgres(Driver):
def __init__(self, data, conf):
super(Postgres, self).__init__(data, conf)
self.dbhost = self.data.get("dbhost", "localhost")
self.dbname = self.data["dbname"]
self.username = self.data.get("username", "postgres")
self.password = self.data.get("password")
self.target_path = str(os.path.join(self.conf.target,
self.host, "pg_dump"))
def snapshot(self):
if self.password:
authline = "{host}:{port}:{dbname}:{username}:{password}".format(
host=self.dbhost, port="5432", dbname=self.dbname,
username=self.username, password=self.password)
home_dir = pwd.getpwuid(os.getuid()).pw_dir
pgpass = os.path.join(home_dir, ".pgpass")
with open(pgpass, "a+") as fo:
fo.seek(0)
auth = False
for line in fo:
if re.search(ur"^{0}$".format(authline), line):
auth = True
break
if not auth:
fo.seek(0, 2)
fo.write("{0}\n".format(authline))
os.chmod(pgpass, stat.S_IRUSR + stat.S_IWUSR)
temp = self.command("mktemp").stdout.strip()
self.command("pg_dump -h {dbhost} -U {username} -w "
"-f {file} {dbname}".format(
dbhost=self.dbhost, username=self.username,
file=temp, dbname=self.dbname))
utils.execute('mkdir -p "{0}"'.format(self.target_path))
dump_basename = "{0}_{1}.sql".format(self.dbhost, self.dbname)
utils.execute('mv -f "{0}" "{1}"'.format(
temp,
os.path.join(self.target_path, dump_basename)))
class XmlRpc(Driver):
def __init__(self, data, conf):
super(XmlRpc, self).__init__(data, conf)
self.server = self.data.get("server", "localhost")
self.methods = self.data.get("methods", [])
self.to_file = self.data.get("to_file")
self.target_path = os.path.join(
self.conf.target, self.host, "xmlrpc", self.to_file)
def snapshot(self):
utils.execute('mkdir -p "{0}"'.format(os.path.dirname(
self.target_path)))
server = xmlrpclib.Server(self.server)
with open(self.target_path, "w") as f:
for method in self.methods:
if hasattr(server, method):
response = getattr(server, method)()
response = pprint.pformat(response, indent=2)
else:
response = "no such method on remote server"
f.write("===== {0} =====\n{1}\n\n".format(method, response))
class Command(Driver):
def __init__(self, data, conf):
super(Command, self).__init__(data, conf)
self.cmdname = self.data["command"]
self.to_file = self.data["to_file"]
self.target_path = os.path.join(
self.conf.target, self.host, "commands", self.to_file)
def snapshot(self):
out = self.command(self.cmdname)
utils.execute('mkdir -p "{0}"'.format(os.path.dirname(
self.target_path)))
with open(self.target_path, "w") as f:
f.write("===== COMMAND =====: {0}\n".format(self.cmdname))
f.write("===== RETURN CODE =====: {0}\n".format(out.return_code))
f.write("===== STDOUT =====:\n")
if out.stdout:
f.write(out.stdout)
f.write("\n===== STDERR =====:\n")
if out.stderr:
f.write(out.stderr)

View File

@ -1,37 +0,0 @@
# Copyright 2013 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import logging
from shotgun.settings import LOG_FILE
def configure_logger():
"""Configures shotgun logger"""
logger = logging.getLogger('shotgun')
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s %(levelname)s %(process)d (%(module)s) %(message)s',
"%Y-%m-%d %H:%M:%S")
stream_handler = logging.StreamHandler()
stream_handler.setLevel(logging.DEBUG)
stream_handler.setFormatter(formatter)
file_handler = logging.FileHandler(LOG_FILE)
file_handler.setLevel(logging.DEBUG)
file_handler.setFormatter(formatter)
logger.addHandler(stream_handler)
logger.addHandler(file_handler)

View File

@ -1,43 +0,0 @@
# Copyright 2013 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import logging
import os
from shotgun.driver import Driver
from shotgun import utils
logger = logging.getLogger(__name__)
class Manager(object):
def __init__(self, conf):
logger.debug("Initializing snapshot manager")
self.conf = conf
def snapshot(self):
logger.debug("Making snapshot")
utils.execute("rm -rf {0}".format(os.path.dirname(self.conf.target)))
for obj_data in self.conf.objects:
logger.debug("Dumping: %s", obj_data)
driver = Driver.getDriver(obj_data, self.conf)
driver.snapshot()
logger.debug("Archiving dump directory: %s", self.conf.target)
utils.compress(self.conf.target, self.conf.compression_level)
with open(self.conf.lastdump, "w") as fo:
fo.write("{0}.tar.xz".format(self.conf.target))
return "{0}.tar.xz".format(self.conf.target)

View File

@ -1,20 +0,0 @@
# Copyright 2013 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
TARGET = "/tmp/snapshot"
LASTDUMP = "/tmp/snapshot_last"
TIMESTAMP = True
COMPRESSION_LEVEL = 3
LOG_FILE = "/var/log/shotgun.log"
DEFAULT_TIMEOUT = 10

View File

@ -1,13 +0,0 @@
# Copyright 2013 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

View File

@ -1,19 +0,0 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from unittest2.case import TestCase
class BaseTestCase(TestCase):
"""Base unit test case for shotgun tests."""

View File

@ -1,58 +0,0 @@
# Copyright 2013 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import time
import mock
from shotgun.config import Config
from shotgun.test import base
class TestConfig(base.BaseTestCase):
def test_timestamp(self):
t = time.localtime()
with mock.patch('shotgun.config.time') as MockedTime:
MockedTime.localtime.return_value = t
MockedTime.strftime.side_effect = time.strftime
conf = Config({})
stamped = conf._timestamp("sample")
self.assertEqual(
stamped,
"sample-{0}".format(time.strftime('%Y-%m-%d_%H-%M-%S', t))
)
def test_target_timestamp(self):
conf = Config({
"target": "/tmp/sample",
"timestamp": True
})
self.assertRegex(
conf.target,
ur"\/tmp\/sample\-[\d]{4}\-[\d]{2}\-[\d]{2}_"
"([\d]{2}\-){2}[\d]{2}",
)
@mock.patch('shotgun.config.settings')
def test_timeout(self, m_settings):
conf = Config({})
self.assertIs(conf.timeout, m_settings.DEFAULT_TIMEOUT)
def test_pass_default_timeout(self):
timeout = 1345
conf = Config({
'timeout': timeout,
})
self.assertEqual(conf.timeout, timeout)

View File

@ -1,219 +0,0 @@
# Copyright 2013 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import random
import sys
import fabric
import mock
import shotgun
from shotgun.test import base
class RunOut(object):
return_code = None
stderr = None
stdout = None
def __str__(self):
return str(self.stdout)
class TestDriver(base.BaseTestCase):
def test_driver_factory(self):
types = {
"file": "File",
"dir": "Dir",
"postgres": "Postgres",
"command": "Command"
}
for t, n in types.iteritems():
with mock.patch("shotgun.driver.%s" % n) as mocked:
shotgun.driver.Driver.getDriver({"type": t}, None)
mocked.assert_called_with({"type": t}, None)
@mock.patch('shotgun.driver.utils.CCStringIO')
@mock.patch('shotgun.driver.fabric.api.settings')
@mock.patch('shotgun.driver.fabric.api.run')
def test_driver_remote_command(self, mfabrun, mfabset, mccstring):
out = shotgun.driver.CommandOut()
out.stdout = "STDOUT"
out.return_code = "RETURN_CODE"
mccstring.return_value.getvalue.return_value = out.stdout
runout = RunOut()
runout.return_code = "RETURN_CODE"
mfabrun.return_value = runout
command = "COMMAND"
conf = mock.Mock()
driver = shotgun.driver.Driver(
{"host": {"address": "remote_host"}}, conf)
result = driver.command(command)
mfabrun.assert_called_with(
command, stdout=mock.ANY)
mfabset.assert_called_with(
host_string="remote_host",
timeout=2,
command_timeout=driver.timeout,
warn_only=True,
key_filename=None,
abort_on_prompts=True)
self.assertEqual(result, out)
@mock.patch('shotgun.driver.fabric.api.run')
@mock.patch('shotgun.driver.fabric.api.settings')
def test_fabric_use_timout_from_driver(self, mfabset, _):
timeout = random.randint(1, 100)
conf = mock.Mock()
driver = shotgun.driver.Driver(
{"host": {"address": "remote_host"}}, conf)
driver.timeout = timeout
driver.command("COMMAND")
mfabset.assert_called_with(
host_string=mock.ANY,
timeout=mock.ANY,
command_timeout=timeout,
warn_only=mock.ANY,
key_filename=mock.ANY,
abort_on_prompts=mock.ANY)
@mock.patch('shotgun.driver.utils.execute')
def test_driver_local_command(self, mexecute):
mexecute.return_value = ("RETURN_CODE", "STDOUT", "STDERR")
out = shotgun.driver.CommandOut()
out.stdout = "STDOUT"
out.stderr = "STDERR"
out.return_code = "RETURN_CODE"
command = "COMMAND"
conf = mock.Mock()
driver = shotgun.driver.Driver({}, conf)
result = driver.command(command)
shotgun.driver.utils.execute.assert_called_with(command)
self.assertEqual(result, out)
@mock.patch('shotgun.driver.utils.CCStringIO')
@mock.patch('shotgun.driver.fabric.api.settings')
@mock.patch('shotgun.driver.fabric.api.run')
def test_command_timeout(self, mfabrun, mfabset, mstringio):
mfabrun.side_effect = fabric.exceptions.CommandTimeout(10)
mstdout = mock.MagicMock()
mstdout.getvalue.return_value = 'FULL STDOUT'
mstringio.return_value = mstdout
command = "COMMAND"
conf = mock.Mock()
driver = shotgun.driver.Driver(
{"host": {"address": "remote_host"}}, conf)
result = driver.command(command)
mstringio.assert_has_calls([
mock.call(writers=sys.stdout),
])
mfabrun.assert_called_with(command, stdout=mstdout)
self.assertEqual(result.stdout, 'FULL STDOUT')
@mock.patch('shotgun.driver.utils.execute')
@mock.patch('shotgun.driver.fabric.api.settings')
@mock.patch('shotgun.driver.fabric.api.get')
def test_driver_get(self, mfabget, mfabset, mexecute):
mexecute.return_value = ("RETURN_CODE", "STDOUT", "STDERR")
remote_path = "/remote_dir/remote_file"
target_path = "/target_dir"
conf = mock.Mock()
driver = shotgun.driver.Driver({
"host": {
"address": "remote_host",
"ssh-key": "path_to_key",
}
}, conf)
driver.get(remote_path, target_path)
mexecute.assert_called_with('mkdir -p "{0}"'.format(target_path))
mfabget.assert_called_with(remote_path, target_path)
mfabset.assert_called_with(
host_string="remote_host", key_filename="path_to_key",
timeout=2, warn_only=True, abort_on_prompts=True)
mexecute.reset_mock()
driver = shotgun.driver.Driver({}, conf)
driver.get(remote_path, target_path)
self.assertEqual(mexecute.mock_calls, [
mock.call('mkdir -p "{0}"'.format(target_path)),
mock.call('cp -r "{0}" "{1}"'.format(remote_path, target_path))])
def test_use_timeout_from_global_conf(self):
data = {}
conf = mock.Mock(spec=shotgun.config.Config, target="some_target")
cmd_driver = shotgun.driver.Driver(data, conf)
self.assertEqual(cmd_driver.timeout, conf.timeout)
def test_use_command_specific_timeout(self):
timeout = 1234
data = {
"timeout": timeout
}
conf = mock.Mock(spec=shotgun.config.Config, target="some_target")
cmd_driver = shotgun.driver.Driver(data, conf)
self.assertEqual(cmd_driver.timeout, timeout)
self.assertNotEqual(cmd_driver.timeout, conf.timeout)
class TestFile(base.BaseTestCase):
@mock.patch('shotgun.driver.Driver.get')
def test_snapshot(self, mget):
data = {
"type": "file",
"path": "/remote_dir/remote_file",
"host": {
"address": "remote_host",
},
}
conf = mock.MagicMock()
conf.target = "/target"
file_driver = shotgun.driver.File(data, conf)
target_path = "/target/remote_host/remote_dir"
file_driver.snapshot()
mget.assert_called_with(data["path"], target_path)
@mock.patch('shotgun.driver.utils.remove')
@mock.patch('shotgun.driver.Driver.get')
def test_dir_exclude_called(self, mget, mremove):
data = {
"type": "dir",
"path": "/remote_dir/",
"exclude": ["*test"],
"host": {
"address": "remote_host",
},
}
conf = mock.MagicMock()
conf.target = "/target"
dir_driver = shotgun.driver.Dir(data, conf)
target_path = "/target/remote_host/remote_dir"
dir_driver.snapshot()
mget.assert_called_with(data["path"], target_path)
mremove.assert_called_with(dir_driver.full_dst_path, data['exclude'])

View File

@ -1,43 +0,0 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import tempfile
import mock
from shotgun.manager import Manager
from shotgun.test import base
class TestManager(base.BaseTestCase):
@mock.patch('shotgun.manager.Driver.getDriver')
@mock.patch('shotgun.manager.utils.execute')
@mock.patch('shotgun.manager.utils.compress')
def test_snapshot(self, mcompress, mexecute, mget):
data = {
"type": "file",
"path": "/remote_dir/remote_file",
"host": {
"address": "remote_host",
},
}
conf = mock.MagicMock()
conf.target = "/target/data"
conf.objects = [data]
conf.lastdump = tempfile.mkstemp()[1]
manager = Manager(conf)
manager.snapshot()
mget.assert_called_once_with(data, conf)
mexecute.assert_called_once_with('rm -rf /target')

View File

@ -1,112 +0,0 @@
# coding: utf-8
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import StringIO
import mock
from shotgun.test import base
from shotgun import utils
class TestUtils(base.BaseTestCase):
@mock.patch('shotgun.utils.execute')
def test_remove_subdir(self, mexecute):
utils.remove('/', ['good', '**/*.py'])
mexecute.assert_has_calls([
mock.call('shopt -s globstar; rm -rf /good', shell=True),
mock.call('shopt -s globstar; rm -rf /**/*.py', shell=True)])
@mock.patch('shotgun.utils.os.walk')
def test_iterfiles(self, mwalk):
path = '/root'
mwalk.return_value = [
(path, '', ('file1', 'file2')),
(path + '/sub', '', ('file3',))]
result = list(utils.iterfiles(path))
mwalk.assert_called_once_with(path, topdown=True)
self.assertEqual(
result, ['/root/file1', '/root/file2', '/root/sub/file3'])
@mock.patch('shotgun.utils.execute')
def test_compress(self, mexecute):
target = '/path/target'
level = '-3'
utils.compress(target, level)
compress_call = mexecute.call_args_list[0]
rm_call = mexecute.call_args_list[1]
compress_env = compress_call[1]['env']
self.assertEqual(compress_env['XZ_OPT'], level)
self.assertEqual(
compress_call[0][0],
'tar cJvf /path/target.tar.xz -C /path target')
self.assertEqual(rm_call[0][0], 'rm -r /path/target')
class TestCCStringIO(base.BaseTestCase):
def test_no_writers(self):
test_string = 'some_string'
ccstring = utils.CCStringIO()
ccstring.write(test_string)
self.assertEqual(ccstring.getvalue(), test_string)
def test_with_one_writer(self):
test_string = 'some_string'
writer = StringIO.StringIO()
ccstring = utils.CCStringIO(writers=writer)
ccstring.write(test_string)
self.assertEqual(ccstring.getvalue(), test_string)
self.assertEqual(writer.getvalue(), test_string)
def test_with_multiple_writers(self):
test_string = 'some_string'
writer_a = StringIO.StringIO()
writer_b = StringIO.StringIO()
ccstring = utils.CCStringIO(writers=[writer_a, writer_b])
ccstring.write(test_string)
self.assertEqual(ccstring.getvalue(), test_string)
self.assertEqual(writer_a.getvalue(), test_string)
self.assertEqual(writer_b.getvalue(), test_string)
def test_with_writer_and_buffer(self):
buffer = 'I am here already'
writer = StringIO.StringIO()
ccstring = utils.CCStringIO(buffer, writers=writer)
self.assertEqual(ccstring.getvalue(), buffer)
self.assertEqual(writer.getvalue(), '')
def test_non_ascii_output_with_unicode(self):
ccstring = utils.CCStringIO()
ccstring.write('привет')
ccstring.write(u'test')
self.assertEqual(ccstring.getvalue(), 'приветtest')

View File

@ -1,158 +0,0 @@
# Copyright 2013 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import copy
import logging
import os
import re
import shlex
import socket
from StringIO import StringIO
import subprocess
logger = logging.getLogger(__name__)
def hostname():
return socket.gethostname()
def is_ip(name):
return (re.search(ur"([0-9]{1,3}\.){3}[0-9]{1,3}", name) and True)
def fqdn(name=None):
if name:
return socket.getfqdn(name)
return socket.getfqdn(socket.gethostname())
def is_local(name):
if name in ("localhost", hostname(), fqdn()):
return True
return False
def iterfiles(path):
for root, dirnames, filenames in os.walk(path, topdown=True):
for filename in filenames:
yield os.path.join(root, filename)
def remove(full_dst_path, excludes):
"""Removes subdirs/files using unixs syntax
full_dst_path is treated as root directory for remove
:param full_dst_path: str
:param excludes: list with excludes paths/files
"""
for exclude in excludes:
path = os.path.join(full_dst_path, exclude.lstrip('/'))
logger.debug('Deleting %s', path)
execute("shopt -s globstar; rm -rf {0}".format(path), shell=True)
def compress(target, level, keep_target=False):
"""Runs compression of provided directory
:param target: directory to compress
:param level: level of compression
:param keep_target: bool, if True target directory wont be removed
"""
env = copy.deepcopy(os.environ)
env['XZ_OPT'] = level
execute("tar cJvf {0}.tar.xz -C {1} {2}"
"".format(target,
os.path.dirname(target),
os.path.basename(target)),
env=env)
if not keep_target:
execute("rm -r {0}".format(target))
def execute(command, to_filename=None, env=None, shell=False):
logger.debug("Trying to execute command: %s", command)
commands = [c.strip() for c in re.split(ur'\|', command)]
env = env or os.environ
env["PATH"] = "/bin:/usr/bin:/sbin:/usr/sbin"
to_file = None
if to_filename:
to_file = open(to_filename, 'wb')
process = []
for c in commands:
try:
# NOTE(eli): Python's shlex implementation doesn't like unicode.
# We have to convert to ascii before shlex'ing the command.
# http://bugs.python.org/issue6988
encoded_command = c.encode('ascii')
process.append(subprocess.Popen(
shlex.split(encoded_command) if not shell else encoded_command,
env=env,
stdin=(process[-1].stdout if process else None),
stdout=(to_file
if (len(process) == len(commands) - 1) and to_file
else subprocess.PIPE),
stderr=(subprocess.PIPE),
shell=shell
))
except OSError as e:
return (1, "", "{0}\n".format(e))
if len(process) >= 2:
process[-2].stdout.close()
stdout, stderr = process[-1].communicate()
return (process[-1].returncode, stdout, stderr)
class CCStringIO(StringIO):
"""A "carbon copy" StringIO.
It's capable of multiplexing its writes to other buffer objects.
Taken from fabric.tests.mock_streams.CarbonCopy
"""
def __init__(self, buffer='', writers=None):
"""CCStringIO initializator
If ``writers`` is given and is a file-like object or an
iterable of same, it/they will be written to whenever this
StringIO instance is written to.
"""
StringIO.__init__(self, buffer)
if writers is None:
writers = []
elif hasattr(writers, 'write'):
writers = [writers]
self.writers = writers
def write(self, s):
# unfortunately, fabric writes into StringIO both so-called
# bytestrings and unicode strings. obviously, bytestrings may
# contain non-ascii symbols. that leads to type-conversion
# issue when we use string's join (inside getvalue()) with
# a list of both unicodes and bytestrings. in order to avoid
# this issue we should convert all input unicode strings into
# utf-8 bytestrings (let's assume that slaves encoding is utf-8
# too so we won't have encoding mess in the output file).
if isinstance(s, unicode):
s = s.encode('utf-8')
StringIO.write(self, s)
for writer in self.writers:
writer.write(s)

View File

@ -1,40 +0,0 @@
%define name shotgun
%{!?version: %define version 8.0.0}
%{!?release: %define release 1}
Name: %{name}
Summary: Shotgun package
Version: %{version}
Release: %{release}
URL: http://mirantis.com
License: Apache
Group: Development/Libraries
BuildRoot: %{_tmppath}/%{name}-%{version}-buildroot
Prefix: %{_prefix}
BuildArch: noarch
Requires: postgresql
Requires: python-fabric >= 1.10.0
Requires: python-argparse
Requires: tar
Requires: gzip
Requires: bzip2
Requires: openssh-clients
Requires: xz
%description
Shotgun package.
%prep
%setup -cq -n %{name}-%{version}
%build
cd %{_builddir}/%{name}-%{version}/shotgun && python setup.py build
%install
cd %{_builddir}/%{name}-%{version}/shotgun && python setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=%{_builddir}/%{name}-%{version}/shotgun/INSTALLED_FILES
%clean
rm -rf $RPM_BUILD_ROOT
%files -f %{_builddir}/%{name}-%{version}/shotgun/INSTALLED_FILES
%defattr(-,root,root)

View File

@ -1,4 +0,0 @@
mock==1.3.0
unittest2==1.1.0
pytest==2.8.0
pytest-cov==2.1.0

View File

@ -1,43 +0,0 @@
[tox]
minversion = 1.6
skipsdist = True
envlist = py26,py27,pep8
[testenv]
usedevelop = True
install_command = pip install --allow-external -U {opts} {packages}
setenv = VIRTUAL_ENV={envdir}
deps = -r{toxinidir}/test-requirements.txt
commands =
py.test -vv {posargs:shotgun/test}
[tox:jenkins]
downloadcache = ~/cache/pip
[testenv:pep8]
deps = hacking==0.10
usedevelop = False
commands =
flake8 {posargs:shotgun}
[testenv:cover]
setenv = VIRTUAL_ENV={envdir}
commands =
py.test -vv --cov=shotgun {posargs:shotgun/test}
[testenv:venv]
commands = {posargs:}
[testenv:devenv]
envdir = devenv
usedevelop = True
[flake8]
ignore = H234,H302,H802
exclude = .venv,.git,.tox,dist,doc,*lib/python*,*egg,build,tools,__init__.py,docs
show-pep8 = True
show-source = True
count = True
[hacking]
import_exceptions = testtools.matchers

View File

@ -117,13 +117,11 @@ cd %{_builddir}/%{name}-%{version}/nailgun && %{_builddir}/%{name}-%{version}/na
mv %{_builddir}/%{name}-%{version}/nailgun/compressed_static %{_builddir}/%{name}-%{version}/nailgun/static
cd %{_builddir}/%{name}-%{version}/nailgun && python setup.py build
cd %{_builddir}/%{name}-%{version}/network_checker && python setup.py build
cd %{_builddir}/%{name}-%{version}/shotgun && python setup.py build
cd %{_builddir}/%{name}-%{version}/fuel_upgrade_system/fuel_package_updates && python setup.py build
%install
cd %{_builddir}/%{name}-%{version}/nailgun && python setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=%{_builddir}/%{name}-%{version}/nailgun/INSTALLED_FILES
cd %{_builddir}/%{name}-%{version}/network_checker && python setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=%{_builddir}/%{name}-%{version}/network_checker/INSTALLED_FILES
cd %{_builddir}/%{name}-%{version}/shotgun && python setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=%{_builddir}/%{name}-%{version}/shotgun/INSTALLED_FILES
cd %{_builddir}/%{name}-%{version}/fuel_upgrade_system/fuel_package_updates && python setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=%{_builddir}/%{name}-%{version}/fuel_upgrade_system/fuel_package_updates/INSTALLED_FILES
mkdir -p %{buildroot}/opt/nailgun/bin
mkdir -p %{buildroot}/etc/cron.d
@ -185,31 +183,6 @@ between hosts in network.
%files -n nailgun-net-check -f %{_builddir}/%{name}-%{version}/network_checker/INSTALLED_FILES
%defattr(-,root,root)
%package -n shotgun
Summary: Shotgun package
Version: %{version}
Release: %{release}
URL: http://mirantis.com
License: Apache
Group: Development/Libraries
BuildRoot: %{_tmppath}/%{name}-%{version}-buildroot
Prefix: %{_prefix}
BuildArch: noarch
Requires: postgresql
Requires: python-fabric >= 1.10.0
Requires: python-argparse
Requires: tar
Requires: gzip
Requires: bzip2
Requires: openssh-clients
Requires: xz
%description -n shotgun
Shotgun package.
%files -n shotgun -f %{_builddir}/%{name}-%{version}/shotgun/INSTALLED_FILES
%defattr(-,root,root)
%package -n fencing-agent
Summary: Fencing agent