diff --git a/kolla_mesos/commands.py b/kolla_mesos/commands.py index ef457012..2d7ace9d 100644 --- a/kolla_mesos/commands.py +++ b/kolla_mesos/commands.py @@ -118,15 +118,15 @@ def get_status(tasks): task1: { 'register': (register_path, reg_status) 'requirements': { - reqt1_path: reqt_status - reqt2_path: reqt_status + reqt1_path: req_status + reqt2_path: req_status ... } } Where: reg_status = 'done', 'running', 'waiting' - reqt_status = '', 'done' + req_status = '', 'done' """ status = {} with zk_utils.connection() as zk: @@ -135,20 +135,20 @@ def get_status(tasks): status[task] = {} status[task]['requirements'] = {} for path in info['requires']: - reqt_status = '' + req_status = ''.encode('utf-8') if zk.exists(path): - reqt_status, _ = zk.get(path) - status[task]['requirements'][path] = reqt_status + req_status, _ = zk.get(path) + status[task]['requirements'][path] = req_status.decode('utf-8') # get status of registrations for task, info in tasks.items(): status[task]['register'] = {} reg_path = info['register'] - reg_status = '' + reg_status = ''.encode('utf-8') if zk.exists(reg_path): reg_status, _ = zk.get(reg_path) - status[task]['register'] = (reg_path, reg_status) + status[task]['register'] = (reg_path, reg_status.decode('utf-8')) return status diff --git a/kolla_mesos/common/cli_utils.py b/kolla_mesos/common/cli_utils.py index f626ee28..10e3d562 100644 --- a/kolla_mesos/common/cli_utils.py +++ b/kolla_mesos/common/cli_utils.py @@ -44,4 +44,4 @@ def dict2columns(data, id_col=None): items = [(key, data[key]) for key in keys] else: items = sorted(data.items()) - return zip(*items) + return list(zip(*items)) diff --git a/kolla_mesos/common/zk_utils.py b/kolla_mesos/common/zk_utils.py index e979984f..b1be518b 100644 --- a/kolla_mesos/common/zk_utils.py +++ b/kolla_mesos/common/zk_utils.py @@ -33,7 +33,7 @@ def _list_all(path, zk): values = {} data, stat = zk.get(path) if stat.dataLength > 0: - values[path] = data + values[path] = data.decode('utf-8') try: children = zk.get_children(path) except exceptions.NoNodeError: @@ -55,12 +55,12 @@ def list_all(path): def get_one(path): with connection() as zk: data, stat = zk.get(path) - return {path: data} + return {path: data.decode('utf-8')} def set_one(path, value): with connection() as zk: - zk.set(path, value) + zk.set(path, value.encode('utf-8')) @contextlib.contextmanager diff --git a/kolla_mesos/configuration.py b/kolla_mesos/configuration.py index 7a4ac5c7..c30f1fe5 100644 --- a/kolla_mesos/configuration.py +++ b/kolla_mesos/configuration.py @@ -49,7 +49,9 @@ def write_variables_zookeeper(zk, variables, base_node=None, overwrite=True): var_path) return zk.ensure_path(var_path) - zk.set(var_path, "" if var_value is None else var_value) + if var_value is None: + var_value = '' + zk.set(var_path, var_value.encode('utf-8')) LOG.debug('Updated "%s" node in zookeeper.' % var_path) @@ -86,7 +88,7 @@ def write_common_config_to_zookeeper(config_dir, zk, jinja_vars, src_file = file_utils.find_file(source_path) with open(src_file) as fp: content = fp.read() - zk.set(script_node, content) + zk.set(script_node, content.encode('utf-8')) def get_variables_from_zookeeper(zk, needed_variables): diff --git a/kolla_mesos/container_scripts/start.py b/kolla_mesos/container_scripts/start.py index f410407a..7ef42d5c 100644 --- a/kolla_mesos/container_scripts/start.py +++ b/kolla_mesos/container_scripts/start.py @@ -288,7 +288,7 @@ def write_file(conf, data): perm = int(conf.get('perm', 0)) with tempfile.NamedTemporaryFile(prefix='kolla-mesos', delete=False) as tf: - tf.write(data) + tf.write(data.encode('utf-8')) tf.flush() tf_name = tf.name @@ -334,10 +334,10 @@ def render_template(zk, templ, variables, var_names): value = '' LOG.warning('missing required variable value %s', var) except kz_exceptions.NoNodeError: - value = '' + value = ''.encode('utf-8') LOG.error('missing required variable %s', var) - variables[var] = value.encode('utf-8') + variables[var] = value.decode('utf-8') return jinja_render(templ, variables) @@ -420,20 +420,20 @@ class Command(object): for check_path in self.check_paths: self.zk.retry(self.zk.ensure_path, check_path) current_state, _ = self.zk.get(check_path) - if current_state != state: + if current_state.decode('utf-8') != state: LOG.info('path: %s, changing state from %s to %s' % (check_path, current_state, state)) - self.zk.set(check_path, state) + self.zk.set(check_path, state.encode('utf-8')) def get_state(self, path=None): if not path: path = self.check_paths[0] state = None if self.zk.exists(path): - state, _ = self.zk.get(str(path)) + state, _ = self.zk.get(path) if not state: - state = None - return state + return None + return state.decode('utf-8') def sleep(self, queue_size, retry=False): seconds = math.ceil(20 / (1.0 + queue_size)) @@ -505,7 +505,7 @@ class Command(object): continue raw_content, stat = self.zk.get(os.path.join(SERVICE, 'files', name)) - templ = raw_content.encode('utf-8') + templ = raw_content.decode('utf-8') var_names = jinja_find_required_variables(templ, name) if not var_names: # not a template, doesn't need rendering. @@ -623,7 +623,7 @@ def main(): LOG.info('starting') with zk_connection(ZK_HOSTS) as zk: service_conf_raw, stat = zk.get(SERVICE) - service_conf = json.loads(service_conf_raw) + service_conf = json.loads(service_conf_raw.decode('utf-8')) # don't join a Party if this container is not running a daemon # process. diff --git a/kolla_mesos/service.py b/kolla_mesos/service.py index a0a0c961..9fb0e77e 100644 --- a/kolla_mesos/service.py +++ b/kolla_mesos/service.py @@ -73,7 +73,7 @@ class File(object): src_file = file_utils.find_file(src_file) with open(src_file) as fp: content = fp.read() - zk.set(dest_node, content) + zk.set(dest_node, content.encode('utf-8')) class Command(object): @@ -125,7 +125,8 @@ class Runner(object): raise exception.KollaNotFoundException( service_name, entity='running service definition') return Runner(yaml.load( - jinja_utils.jinja_render_str(conf_raw, variables))) + jinja_utils.jinja_render_str(conf_raw.decode('utf-8'), + variables))) @classmethod def load_from_file(cls, service_file, variables): @@ -148,7 +149,7 @@ class Runner(object): dest_node = os.path.join(base_node, self._conf['name']) zk.ensure_path(dest_node) try: - zk.set(dest_node, json.dumps(self._conf)) + zk.set(dest_node, json.dumps(self._conf).encode('utf-8')) except Exception as te: LOG.error('%s=%s -> %s' % (dest_node, self._conf, te)) @@ -444,7 +445,8 @@ def _load_variables_from_zk(zk): except exceptions.NoNodeError: var_names = [] for var in var_names: - variables[str(var)], _stat = zk.get(os.path.join(path, var)) + value, _stat = zk.get(os.path.join(path, var)) + variables[var] = value.decode('utf-8') # Add deployment_id variables.update({'deployment_id': CONF.kolla.deployment_id}) # override node_config_directory to empty diff --git a/kolla_mesos/tests/common/test_zk_utils.py b/kolla_mesos/tests/common/test_zk_utils.py index fc05c12e..f4e68758 100644 --- a/kolla_mesos/tests/common/test_zk_utils.py +++ b/kolla_mesos/tests/common/test_zk_utils.py @@ -28,9 +28,9 @@ class TestConfig(base.BaseTestCase): def test_list_all(self): self.client.create('/kolla/t1/status/q/x', - 'val-1', makepath=True) + 'val-1'.encode('utf-8'), makepath=True) self.client.create('/kolla/t1/variables/x', - 'val-2', makepath=True) + 'val-2'.encode('utf-8'), makepath=True) with mock.patch.object(zk_utils, 'connection') as m_zk_c: @@ -41,7 +41,7 @@ class TestConfig(base.BaseTestCase): def test_get_one(self): self.client.create('/kolla/t1/variables/x', - 'val', makepath=True) + 'val'.encode('utf-8'), makepath=True) with mock.patch.object(zk_utils, 'connection') as m_zk_c: @@ -51,11 +51,11 @@ class TestConfig(base.BaseTestCase): def test_set_one(self): self.client.create('/kolla/t1/variables/x', - 'old', makepath=True) + 'old'.encode('utf-8'), makepath=True) with mock.patch.object(zk_utils, 'connection') as m_zk_c: m_zk_c.return_value.__enter__.return_value = self.client zk_utils.set_one('/kolla/t1/variables/x', 'new') val, _st = self.client.get('/kolla/t1/variables/x') - self.assertEqual('new', val) + self.assertEqual('new', val.decode('utf-8')) diff --git a/kolla_mesos/tests/container_scripts/test_start.py b/kolla_mesos/tests/container_scripts/test_start.py index 60efaa02..a978cd5e 100644 --- a/kolla_mesos/tests/container_scripts/test_start.py +++ b/kolla_mesos/tests/container_scripts/test_start.py @@ -16,6 +16,7 @@ from kazoo.recipe import party import logging import mock import os.path +import six import sys from zake import fake_client @@ -86,7 +87,7 @@ class CommandTest(base.BaseTestCase): self.client) self.client.create('/kolla/t1/status/q/x', - 'done', makepath=True) + 'done'.encode('utf-8'), makepath=True) self.assertFalse(cmd1.requirements_fulfilled()) @@ -103,9 +104,9 @@ class CommandTest(base.BaseTestCase): self.client) self.client.create('/kolla/t1/status/global/w/x', - 'done', makepath=True) + 'done'.encode('utf-8'), makepath=True) self.client.create('/kolla/t1/status/test-hostname/y/l', - 'done', makepath=True) + 'done'.encode('utf-8'), makepath=True) self.assertTrue(cmd1.requirements_fulfilled()) @mock.patch('socket.gethostname') @@ -135,7 +136,7 @@ class CommandTest(base.BaseTestCase): self.client) self.client.create('/kolla/t1/status/test-hostname/testr/a', - 'done', makepath=True) + 'done'.encode('utf-8'), makepath=True) self.assertEqual(None, cmd1.get_state( path='/kolla/t1/status/global/testr/a')) @@ -200,8 +201,12 @@ class CommandTest(base.BaseTestCase): call_order.append('get_state') return start.CMD_DONE - def lock_side_effect(): - call_order.append('lock_enter') + if six.PY3: + def lock_side_effect(lck_self): + call_order.append('lock_enter') + else: + def lock_side_effect(): + call_order.append('lock_enter') m_lock = mock.MagicMock() with mock.patch.object(self.client, 'Lock', m_lock): @@ -383,7 +388,8 @@ class GenerateConfigTest(base.BaseTestCase): 'command': 'true', 'files': afile}}} m_gar.return_value = {}, {} - self.client.create('/kolla/deploy_id/testg/testr/files/afile', 'xyz', + self.client.create('/kolla/deploy_id/testg/testr/files/afile', + 'xyz'.encode('utf-8'), makepath=True) start.run_commands(self.client, conf) m_wf.assert_called_once_with(afile['afile'], 'xyz') @@ -401,10 +407,11 @@ class GenerateConfigTest(base.BaseTestCase): conf = {'commands': {'setup': { 'command': 'true', 'files': afile}}} m_gar.return_value = {}, {} - self.client.create('/kolla/deploy_id/variables/xyz', 'yeah', + self.client.create('/kolla/deploy_id/variables/xyz', + 'yeah'.encode('utf-8'), makepath=True) self.client.create('/kolla/deploy_id/testg/testr/files/afile', - '{{ xyz }}', makepath=True) + '{{ xyz }}'.encode('utf-8'), makepath=True) start.run_commands(self.client, conf) m_wf.assert_called_once_with(afile['afile'], 'yeah') @@ -424,7 +431,7 @@ class GenerateConfigTest(base.BaseTestCase): m_gar.return_value = {}, {} m_rt.return_value = '' self.client.create('/kolla/deploy_id/testg/testr/files/afile', - '{{ xyz }}', makepath=True) + '{{ xyz }}'.encode('utf-8'), makepath=True) start.run_commands(self.client, conf) m_wf.assert_called_once_with(afile['afile'], '') @@ -490,7 +497,7 @@ class MainTest(base.BaseTestCase): acmd = {'command': 'true', 'files': {'afile': afile}} tconf = {'commands': {'thing': acmd}} self.client.create('/kolla/deploy_id/testg/testr', - json.dumps(tconf), makepath=True) + json.dumps(tconf).encode('utf-8'), makepath=True) m_zk_c = mock.MagicMock() with mock.patch.object(start, 'zk_connection', m_zk_c): @@ -512,7 +519,7 @@ class MainTest(base.BaseTestCase): acmd = {'command': 'true', 'files': {'afile': afile}} tconf = {'service': {'daemon': acmd}} self.client.create('/kolla/deploy_id/testg/testr', - json.dumps(tconf), makepath=True) + json.dumps(tconf).encode('utf-8'), makepath=True) m_gmc.return_value = tconf m_zk_c = mock.MagicMock() @@ -704,7 +711,7 @@ class RenderNovaConfTest(base.BaseTestCase): 'nova_api_host': 'nova-api-nova-openstack-did.mfrm.mdom'} for nam, val in variables.items(): self.client.create('/kolla/did/variables/%s' % nam, - getattr(self, nam, val), + getattr(self, nam, val).encode('utf-8'), makepath=True) @mock.patch('time.sleep') @@ -742,7 +749,7 @@ class RenderNovaConfTest(base.BaseTestCase): template_contents = nc.read() self.client.create( '/kolla/did/openstack/nova/nova-compute/files/afile', - template_contents, makepath=True) + template_contents.encode('utf-8'), makepath=True) cmp_file = os.path.join(mod_dir, 'nova-%s.conf' % self.out) with open(cmp_file) as cf: diff --git a/kolla_mesos/tests/fakes/config_file.py b/kolla_mesos/tests/fakes/config_file.py index 8ca47f64..1125490c 100644 --- a/kolla_mesos/tests/fakes/config_file.py +++ b/kolla_mesos/tests/fakes/config_file.py @@ -29,11 +29,12 @@ class FakeConfigFile(object): def __enter__(self): if six.PY3: func = 'builtins.open' + return_val = io.StringIO(self.text_config) else: func = '__builtin__.open' + return_val = io.BytesIO(self.text_config) - self.patcher = mock.patch(func, - return_value=io.BytesIO(self.text_config)) + self.patcher = mock.patch(func, return_value=return_val) self.patcher.start() def __exit__(self, *args, **kwargs): diff --git a/kolla_mesos/tests/test_commands.py b/kolla_mesos/tests/test_commands.py index 8d73bba6..be6fd18c 100644 --- a/kolla_mesos/tests/test_commands.py +++ b/kolla_mesos/tests/test_commands.py @@ -82,13 +82,13 @@ class CommandsTest(base.BaseTestCase): } self.client.create( '%s/cinder-api/db_sync' % var, - 'waiting', makepath=True) + 'waiting'.encode('utf-8'), makepath=True) self.client.create( '%s/cinder_ansible_tasks/create_database' % var, - 'running', makepath=True) + 'running'.encode('utf-8'), makepath=True) self.client.create( '%s/cinder_ansible_tasks/database_user_create' % var, - 'done', makepath=True) + 'done'.encode('utf-8'), makepath=True) status = commands.get_status(test_tasks) self.assertEqual({'cinder-api/db_sync': exp}, status) @@ -113,10 +113,10 @@ class CommandsTest(base.BaseTestCase): # create the done states self.client.create( '%s/cinder_ansible_tasks/create_database' % var, - 'done', makepath=True) + 'done'.encode('utf-8'), makepath=True) self.client.create( '%s/cinder_ansible_tasks/database_user_create' % var, - 'done', makepath=True) + 'done'.encode('utf-8'), makepath=True) status = commands.get_status(test_tasks) self.assertEqual({'cinder-api/db_sync': exp}, status) @@ -142,11 +142,12 @@ class CommandsTest(base.BaseTestCase): # create the done state self.client.create( '%s/cinder_ansible_tasks/create_database' % var, - 'done', makepath=True) + 'done'.encode('utf-8'), makepath=True) self.client.create( '%s/cinder_ansible_tasks/database_user_create' % var, - 'done', makepath=True) - self.client.create('%s/cinder-api/db_sync' % var, 'done', + 'done'.encode('utf-8'), makepath=True) + self.client.create('%s/cinder-api/db_sync' % var, + 'done'.encode('utf-8'), makepath=True) status = commands.get_status(test_tasks) diff --git a/kolla_mesos/tests/test_deployment.py b/kolla_mesos/tests/test_deployment.py index 97309e5b..f5209bf4 100644 --- a/kolla_mesos/tests/test_deployment.py +++ b/kolla_mesos/tests/test_deployment.py @@ -29,8 +29,7 @@ class TestWriteOpenRC(base.BaseTestCase): self.addCleanup(self.client.stop) self.addCleanup(self.client.close) - @mock.patch('kolla_mesos.deployment.open') - def test_write_openrc_ok(self, mock_open): + def test_write_openrc_ok(self): variables = {'keystone_admin_password': 'foofee', 'kolla_internal_address': 'here.not', 'keystone_admin_port': '4511', @@ -38,15 +37,18 @@ class TestWriteOpenRC(base.BaseTestCase): 'keystone_auth_host': 'not.here'} configuration.write_variables_zookeeper(self.client, variables) - mock_open.return_value = mock.MagicMock(spec=file) - file_handle = mock_open.return_value.__enter__.return_value - with mock.patch.object(deployment.zk_utils, 'connection') as m_zk_c: - m_zk_c.return_value.__enter__.return_value = self.client - deployment.write_openrc('openrc') + m_open = mock.mock_open() + with mock.patch('kolla_mesos.deployment.open', m_open): + file_handle = m_open.return_value.__enter__.return_value - mock_open.assert_called_once_with('openrc', 'w') - self.assertEqual(1, file_handle.write.call_count) + with mock.patch.object(deployment.zk_utils, + 'connection') as m_zk_c: + m_zk_c.return_value.__enter__.return_value = self.client + deployment.write_openrc('openrc') + + m_open.assert_called_once_with('openrc', 'w') + self.assertEqual(1, file_handle.write.call_count) def test_write_openrc_fail(self): # missing variable "keystone_admin_port" diff --git a/kolla_mesos/tests/test_service.py b/kolla_mesos/tests/test_service.py index 784d598b..2eccb31c 100644 --- a/kolla_mesos/tests/test_service.py +++ b/kolla_mesos/tests/test_service.py @@ -72,7 +72,7 @@ class TestAPI(base.BaseTestCase): def test_kill(self, m_kill): self.client.create('/kolla/did/openstack/nova/nova-api', json.dumps({'name': 'openstack/nova/nova-api', - 'service': {}}), + 'service': {}}).encode('utf-8'), makepath=True) with mock.patch.object(service.zk_utils, @@ -86,7 +86,7 @@ class TestAPI(base.BaseTestCase): def test_get_marathon(self, m_get_state, c_get_state): self.client.create('/kolla/did/openstack/nova/nova-api', json.dumps({'name': 'openstack/nova/nova-api', - 'service': {}}), + 'service': {}}).encode('utf-8'), makepath=True) with mock.patch.object(service.zk_utils, 'connection') as m_zk_c: @@ -100,7 +100,7 @@ class TestAPI(base.BaseTestCase): def test_get_chronos(self, m_get_state, c_get_state): self.client.create('/kolla/did/openstack/nova/nova_init', json.dumps({'name': 'openstack/nova/nova_init', - 'task': {}}), + 'task': {}}).encode('utf-8'), makepath=True) with mock.patch.object(service.zk_utils, 'connection') as m_zk_c: @@ -113,7 +113,7 @@ class TestAPI(base.BaseTestCase): def test_scale_marathon(self, m_scale): self.client.create('/kolla/did/openstack/nova/nova-api', json.dumps({'name': 'openstack/nova/nova-api', - 'service': {}}), + 'service': {}}).encode('utf-8'), makepath=True) with mock.patch.object(service.zk_utils, 'connection') as m_zk_c: @@ -127,7 +127,7 @@ class TestAPI(base.BaseTestCase): def test_update_marathon(self, m_update, m_apply, m_gmf): self.client.create('/kolla/did/openstack/nova/nova-api', json.dumps({'name': 'openstack/nova/nova-api', - 'service': {}}), + 'service': {}}).encode('utf-8'), makepath=True) with mock.patch.object(service.zk_utils, 'connection') as m_zk_c: