openstack
/
shotgun
Code Issues Proposed changes

Avoid usage of `localhost` and `127.0.0.1` 

* Avoid usage of `localhost` and `127.0.0.1`;
* For a remote command execution either `address` or `hostname` could
 be used;
* Execute a command on a shotgun' host if neither `address` nor
  `hostname` is specified;

Change-Id: Iac1d9ea37400e4d202a9bc673c5fa3d3d185701b
Partial-Bug: #1511730
Partial-Bug: #1517844
This commit is contained in:
Andrey Tykhonov 2015-11-03 17:57:16 +02:00
parent c377d16351
commit a3d413d1ca
3 changed files with 108 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
shotgun/driver.py
View File

@ -17,6 +17,7 @@ import os
import pprint
import pwd
import re
import socket
import stat
import sys
import xmlrpclib
@ -60,13 +61,26 @@ class Driver(object):
def __init__(self, data, conf):
logger.debug("Initializing driver %s: host=%s",
self.__class__.__name__, data.get("host"))
self.data = data
self.host = self.data.get("host", {}).get("hostname", "localhost")
self.addr = self.data.get("host", {}).get("address", "127.0.0.1")
self.ssh_key = self.data.get("host", {}).get("ssh-key")
self.local = utils.is_local(self.host)
data_host = data.get("host", {})
hostname = data_host.get("hostname")
address = data_host.get("address")
self.ssh_key = data_host.get("ssh-key")
# `dest_host` is a IP address or a hostname which is used for network
# connection. IP address is preferable for network connection.
self.dest_host = address or hostname # destination host
# `host` is a hostname or an IP address or hostname of the local host
# which is used for reporting (e.g. logging reporting); a hostname is
# more preferred than an address for reporting. If neither hostname nor
# address is used than it is considered that a command (or a file
# operation) will be executed on a local host, so we use local hostname
# for reporting.
self.host = hostname or address or socket.gethostname()
self.conf = conf
self.timeout = self.data.get("timeout", self.conf.timeout)
self.timeout = data.get("timeout", self.conf.timeout)
def snapshot(self):
raise NotImplementedError
@ -76,17 +90,18 @@ class Driver(object):
raw_stdout = utils.CCStringIO(writers=sys.stdout)
try:
if not self.local:
if self.dest_host:
with fabric.api.settings(
host_string=self.addr, # destination host
key_filename=self.ssh_key, # a path to ssh key
timeout=2, # a network connection timeout
host_string=self.dest_host, # destination host
key_filename=self.ssh_key, # a path to ssh key
timeout=2, # connection timeout
command_timeout=self.timeout, # command execution timeout
warn_only=True, # don't exit on error
abort_on_prompts=True, # non-interactive mode
warn_only=True, # don't exit on error
abort_on_prompts=True, # non-interactive mode
):
logger.debug("Running remote command: "
"host: %s command: %s", self.host, command)
logger.debug(
"Running remote command: host: %s command: %s",
self.host, command)
try:
output = fabric.api.run(command, stdout=raw_stdout)
except SystemExit:
@ -115,13 +130,13 @@ class Driver(object):
copied files or directories
"""
try:
if not self.local:
if self.dest_host:
with fabric.api.settings(
host_string=self.addr, # destination host
key_filename=self.ssh_key, # a path to ssh key
timeout=2, # a network connection timeout
warn_only=True, # don't exit on error
abort_on_prompts=True, # non-interactive mode
host_string=self.dest_host, # destination host
key_filename=self.ssh_key, # a path to ssh key
timeout=2, # connection timeout
warn_only=True, # don't exit on error
abort_on_prompts=True, # non-interactive mode
):
logger.debug("Getting remote file: %s %s",
path, target_path)
@ -131,11 +146,11 @@ class Driver(object):
except SystemExit:
logger.error("Fabric aborted this iteration")
else:
logger.debug("Getting local file: cp -r %s %s",
path, target_path)
logger.debug(
"Getting local file: cp -r %s %s", path, target_path)
utils.execute('mkdir -p "{0}"'.format(target_path))
return utils.execute('cp -r "{0}" "{1}"'.format(path,
target_path))
return utils.execute(
'cp -r "{0}" "{1}"'.format(path, target_path))
except fabric.exceptions.NetworkError as e:
logger.error("NetworkError occured: %s", str(e))
raise
@ -144,10 +159,11 @@ class Driver(object):
class File(Driver):
def __init__(self, data, conf):
super(File, self).__init__(data, conf)
self.path = self.data["path"]
self.exclude = self.data.get('exclude', [])
self.path = data["path"]
self.exclude = data.get('exclude', [])
logger.debug("File to get: %s", self.path)
self.target_path = str(os.path.join(
self.conf.target, self.host,
@ -177,10 +193,10 @@ Dir = File
class Postgres(Driver):
def __init__(self, data, conf):
super(Postgres, self).__init__(data, conf)
self.dbhost = self.data.get("dbhost", "localhost")
self.dbname = self.data["dbname"]
self.username = self.data.get("username", "postgres")
self.password = self.data.get("password")
self.dbhost = data.get("dbhost", "localhost")
self.dbname = data["dbname"]
self.username = data.get("username", "postgres")
self.password = data.get("password")
self.target_path = str(os.path.join(self.conf.target,
self.host, "pg_dump"))
@ -219,9 +235,9 @@ class XmlRpc(Driver):
def __init__(self, data, conf):
super(XmlRpc, self).__init__(data, conf)
self.server = self.data.get("server", "localhost")
self.methods = self.data.get("methods", [])
self.to_file = self.data.get("to_file")
self.server = data.get("server", "localhost")
self.methods = data.get("methods", [])
self.to_file = data.get("to_file")
self.target_path = os.path.join(
self.conf.target, self.host, "xmlrpc", self.to_file)
@ -246,8 +262,8 @@ class Command(Driver):
def __init__(self, data, conf):
super(Command, self).__init__(data, conf)
self.cmdname = self.data["command"]
self.to_file = self.data["to_file"]
self.cmdname = data["command"]
self.to_file = data["to_file"]
self.target_path = os.path.join(
self.conf.target, self.host, "commands", self.to_file)
@ -278,5 +294,5 @@ class Offline(Driver):
utils.execute('mkdir -p "{0}"'.format(os.path.dirname(
self.target_path)))
with open(self.target_path, "w") as f:
f.write("Host {0} with IP {1} was offline/unreachable during "
"logs obtaining.\n".format(self.host, self.addr))
f.write("Host {0} was offline/unreachable during "
"logs obtaining.\n".format(self.host))

65
shotgun/test/test_driver.py
View File

@ -60,9 +60,12 @@ class TestDriver(base.BaseTestCase):
command = "COMMAND"
conf = mock.Mock()
driver = shotgun.driver.Driver(
{"host": {"hostname": "remote_host", 'address': '10.109.0.2'}},
conf)
driver = shotgun.driver.Driver({
"host": {
"hostname": "remote_host",
"address": "10.109.0.2"
}
}, conf)
result = driver.command(command)
mfabrun.assert_called_with(
@ -81,9 +84,12 @@ class TestDriver(base.BaseTestCase):
def test_fabric_use_timout_from_driver(self, mfabset, _):
timeout = random.randint(1, 100)
conf = mock.Mock()
driver = shotgun.driver.Driver(
{"host": {"hostname": "remote_host", "address": "10.109.0.2"}},
conf)
driver = shotgun.driver.Driver({
"host": {
"hostname": "remote_host",
"address": "10.109.0.2"
}
}, conf)
driver.timeout = timeout
driver.command("COMMAND")
mfabset.assert_called_with(
@ -123,8 +129,12 @@ class TestDriver(base.BaseTestCase):
command = "COMMAND"
conf = mock.Mock()
driver = shotgun.driver.Driver(
{"host": {"hostname": "remote_host"}}, conf)
driver = shotgun.driver.Driver({
"host": {
"hostname": "remote_host",
"address": "10.109.0.2"
}
}, conf)
result = driver.command(command)
mstringio.assert_has_calls([
@ -147,7 +157,7 @@ class TestDriver(base.BaseTestCase):
"hostname": "remote_host",
"address": "10.109.0.2",
"ssh-key": "path_to_key",
}
},
}, conf)
driver.get(remote_path, target_path)
mexecute.assert_called_with('mkdir -p "{0}"'.format(target_path))
@ -180,6 +190,41 @@ class TestDriver(base.BaseTestCase):
self.assertEqual(cmd_driver.timeout, timeout)
self.assertNotEqual(cmd_driver.timeout, conf.timeout)
def test_host_when_host_is_specified(self):
conf = mock.Mock(spec=shotgun.config.Config, target="some_target")
hostname = 'example.com'
driver = shotgun.driver.Driver({
'host': {
'hostname': hostname
}
}, conf)
self.assertEqual(driver.host, hostname)
def test_host_when_addr_is_specified(self):
conf = mock.Mock(spec=shotgun.config.Config, target="some_target")
address = '198.51.100.2'
driver = shotgun.driver.Driver({
'host': {
'address': address
}
}, conf)
self.assertEqual(driver.host, address)
@mock.patch('shotgun.driver.socket.gethostname')
def test_host_when_neither_addr_nor_hostname_is_specified(
self, mock_hostname):
conf = mock.Mock(spec=shotgun.config.Config, target="some_target")
sentinel = mock.sentinel
mock_hostname.return_value = sentinel
driver = shotgun.driver.Driver({
'host': {}
}, conf)
self.assertEqual(driver.host, sentinel)
self.assertEqual(mock_hostname.call_count, 1)
class TestFile(base.BaseTestCase):
@ -250,7 +295,7 @@ class TestOffline(base.BaseTestCase):
offline_driver.snapshot()
file_handle_mock = mopen.return_value.__enter__.return_value
file_handle_mock.write.assert_called_once_with(
'Host remote_host with IP 10.109.0.2 was offline/unreachable '
'Host remote_host was offline/unreachable '
'during logs obtaining.\n')
mopen.assert_called_once_with(target_path, 'w')
mexec.assert_called_once_with('mkdir -p "/target/remote_host"')

6
shotgun/utils.py
View File

@ -39,12 +39,6 @@ def fqdn(name=None):
return socket.getfqdn(socket.gethostname())
def is_local(name):
if name in ("localhost", hostname(), fqdn()):
return True
return False
def iterfiles(path):
for root, dirnames, filenames in os.walk(path, topdown=True):
for filename in filenames: