openstack
/
python-muranoclient
Code Issues Proposed changes

Add python 3 support 

Use six versions of StringIO, urllib and raw_input
Use BytesIO instead of StringIO, where relevant
Added ability to distinguish binary and non-binary modes in utils.File
Use six.b for lyterals where relevant
Use six.iteritems, where relevant

Targets blueprint: client-py34-support

Depends-On: Id0e92840eb896ed487aafdcc39074aedefb55dda
Change-Id: I78a914cd77da7263d2bd0f871b8a478b5adadaee
This commit is contained in:
Kirill Zaitsev 2015-11-11 16:51:34 +03:00
parent 38973b4958
commit 1627a5a24f
11 changed files with 72 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
muranoclient/common/http.py
View File

@ -24,7 +24,7 @@ from oslo_serialization import jsonutils
from oslo_utils import encodeutils
import requests
import six
from six.moves.urllib import parse
from six.moves import urllib
from muranoclient.common import exceptions as exc
@ -75,7 +75,7 @@ class HTTPClient(object):
}
self.verify_cert = None
if parse.urlparse(endpoint).scheme == "https":
if urllib.parse.urlparse(endpoint).scheme == "https":
if kwargs.get('insecure'):
self.verify_cert = False
else:

41
muranoclient/common/utils.py
View File

@ -19,15 +19,14 @@ import collections
import os
import re
import shutil
import StringIO
import sys
import tempfile
import textwrap
import urlparse
import uuid
import warnings
import zipfile
import json
from oslo_log import log as logging
from oslo_serialization import jsonutils
from oslo_utils import encodeutils
@ -35,6 +34,7 @@ from oslo_utils import importutils
import prettytable
import requests
import six
from six.moves import urllib
import yaml
import yaql
@ -195,10 +195,12 @@ class NoCloseProxy(object):
class File(object):
def __init__(self, name):
def __init__(self, name, binary=True):
self.name = name
self.binary = binary
def open(self):
mode = 'rb' if self.binary else 'r'
if hasattr(self.name, 'read'):
# NOTE(kzaitsev) We do not want to close a file object
# passed to File wrapper. The caller should be responsible
@ -206,25 +208,24 @@ class File(object):
return NoCloseProxy(self.name)
else:
if os.path.isfile(self.name):
return open(self.name)
url = urlparse.urlparse(self.name)
return open(self.name, mode)
url = urllib.parse.urlparse(self.name)
if url.scheme in ('http', 'https'):
resp = requests.get(self.name, stream=True)
if not resp.ok:
raise ValueError("Got non-ok status({0}) "
"while connecting to {1}".format(
resp.status_code, self.name))
temp_file = tempfile.NamedTemporaryFile()
temp_file = tempfile.NamedTemporaryFile(mode='w+b')
for chunk in resp.iter_content(1024 * 1024):
temp_file.write(chunk)
temp_file.flush()
temp_file.seek(0)
return temp_file
return open(temp_file.name, mode)
raise ValueError("Can't open {0}".format(self.name))
def to_url(filename, base_url, version='', path='/', extension=''):
if urlparse.urlparse(filename).scheme in ('http', 'https'):
if urllib.parse.urlparse(filename).scheme in ('http', 'https'):
return filename
if not base_url:
raise ValueError("No base_url for repository supplied")
@ -232,7 +233,8 @@ def to_url(filename, base_url, version='', path='/', extension=''):
raise ValueError("Invalid filename path supplied: {0}".format(
filename))
version = '.' + version if version else ''
return urlparse.urljoin(base_url, path + filename + version + extension)
return urllib.parse.urljoin(base_url,
path + filename + version + extension)
class FileWrapperMixin(object):
@ -253,15 +255,16 @@ class FileWrapperMixin(object):
if self._file and not self._file.closed:
self._file.close()
def save(self, dst):
def save(self, dst, binary=True):
file_name = self.file_wrapper.name
if urlparse.urlparse(file_name).scheme:
if urllib.parse.urlparse(file_name).scheme:
file_name = file_name.split('/')[-1]
dst = os.path.join(dst, file_name)
with open(dst, 'wb') as dst_file:
mode = 'wb' if binary else 'w'
with open(dst, mode) as dst_file:
self._file.seek(0)
shutil.copyfileobj(self._file, dst_file)
@ -316,7 +319,7 @@ class Package(FileWrapperMixin):
try:
self._file.seek(0)
self._zip_obj = zipfile.ZipFile(
StringIO.StringIO(self._file.read()))
six.BytesIO(self._file.read()))
except Exception as e:
LOG.error("Error {0} occurred,"
" while parsing the package".format(e))
@ -386,7 +389,7 @@ class Package(FileWrapperMixin):
dep_dict = {}
dep_dict[self.manifest['FullName']] = self
if 'Require' in self.manifest:
for dep_name, ver in self.manifest['Require'].iteritems():
for dep_name, ver in six.iteritems(self.manifest['Require']):
if dep_name in dep_dict:
continue
try:
@ -419,7 +422,7 @@ def save_image_local(image_spec, base_url, dst):
path='images/'
)
with open(dst, "wb") as image_file:
with open(dst, "w") as image_file:
response = requests.get(download_url, stream=True)
total_length = response.headers.get('content-length')
@ -543,7 +546,7 @@ class Bundle(FileWrapperMixin):
@staticmethod
def from_file(file_obj):
if not isinstance(file_obj, File):
file_obj = File(file_obj)
file_obj = File(file_obj, binary=False)
return Bundle(file_obj)
@staticmethod
@ -558,7 +561,9 @@ class Bundle(FileWrapperMixin):
self._file.seek(0)
bundle = None
try:
bundle = jsonutils.load(self._file)
# NOTE(kzaitsev) jsonutils throws a type error here
# see bug 1515231
bundle = json.load(self._file)
except ValueError:
pass
if bundle is None:

4
muranoclient/common/yaqlexpression.py
View File

@ -12,7 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
import re
import types
import six
import yaql
from yaql.language import exceptions as yaql_exc
@ -43,7 +43,7 @@ class YaqlExpression(object):
@staticmethod
def match(expr):
if not isinstance(expr, types.StringTypes):
if not isinstance(expr, six.string_types):
return False
if re.match('^[\s\w\d.:]*$', expr):
return False

3
muranoclient/tests/unit/test_methods.py
View File

@ -13,6 +13,7 @@
# under the License.
import mock
import six
import testtools
from muranoclient import client
@ -238,7 +239,7 @@ class UnitTestsForClassesAndFunctions(testtools.TestCase):
api.configure_mock(**{'json_request.side_effect': json_request})
manager = packages.PackageManager(api)
category = '\xd0\xbf\xd0\xb8\xd0\xb2\xd0\xbe'
category = six.b('\xd0\xbf\xd0\xb8\xd0\xb2\xd0\xbe')
kwargs = {'category': category.decode('utf-8')}
list(manager.filter(**kwargs))

35
muranoclient/tests/unit/test_shell.py
View File

@ -19,7 +19,6 @@ import logging
import os
import re
import shutil
import StringIO
import sys
import tempfile
@ -531,7 +530,7 @@ class ShellCommandTest(ShellTest):
]),
]
temp_file = tempfile.NamedTemporaryFile(prefix="murano-test")
temp_file = tempfile.NamedTemporaryFile(prefix="murano-test", mode='w')
json.dump([
{'op': 'replace', 'path': '/0/?/name',
'value': "dummy"
@ -721,7 +720,7 @@ class ShellPackagesOperations(ShellCommandTest):
v1_shell.do_package_import(self.client, args)
return f.name
@mock.patch('__builtin__.raw_input')
@mock.patch('six.moves.input')
@mock.patch('muranoclient.common.utils.Package.from_file')
def test_package_import_conflict_skip(self, from_file, raw_input_mock):
@ -736,7 +735,7 @@ class ShellPackagesOperations(ShellCommandTest):
'is_public': False,
}, {name: mock.ANY},)
@mock.patch('__builtin__.raw_input')
@mock.patch('six.moves.input')
@mock.patch('muranoclient.common.utils.Package.from_file')
def test_package_import_conflict_skip_ea(self, from_file, raw_input_mock):
@ -753,7 +752,7 @@ class ShellPackagesOperations(ShellCommandTest):
}, {name: mock.ANY},)
self.assertFalse(raw_input_mock.called)
@mock.patch('__builtin__.raw_input')
@mock.patch('six.moves.input')
@mock.patch('muranoclient.common.utils.Package.from_file')
def test_package_import_conflict_abort(self, from_file, raw_input_mock):
@ -768,7 +767,7 @@ class ShellPackagesOperations(ShellCommandTest):
'is_public': False,
}, mock.ANY,)
@mock.patch('__builtin__.raw_input')
@mock.patch('six.moves.input')
@mock.patch('muranoclient.common.utils.Package.from_file')
def test_package_import_conflict_abort_ea(self,
from_file, raw_input_mock):
@ -786,7 +785,7 @@ class ShellPackagesOperations(ShellCommandTest):
}, mock.ANY,)
self.assertFalse(raw_input_mock.called)
@mock.patch('__builtin__.raw_input')
@mock.patch('six.moves.input')
@mock.patch('muranoclient.common.utils.Package.from_file')
def test_package_import_conflict_update(self, from_file, raw_input_mock):
@ -807,7 +806,7 @@ class ShellPackagesOperations(ShellCommandTest):
)
self.assertEqual(2, self.client.packages.create.call_count)
@mock.patch('__builtin__.raw_input')
@mock.patch('six.moves.input')
@mock.patch('muranoclient.common.utils.Package.from_file')
def test_package_import_conflict_update_ea(self,
from_file, raw_input_mock):
@ -931,13 +930,13 @@ class ShellPackagesOperations(ShellCommandTest):
m.get(TestArgs.murano_repo_url + '/apps/first_app.zip', body=pkg1)
m.get(TestArgs.murano_repo_url + '/apps/second_app.1.0.zip',
body=pkg2)
s = StringIO.StringIO()
s = six.StringIO()
bundle_contents = {'Packages': [
{'Name': 'first_app'},
{'Name': 'second_app', 'Version': '1.0'}
]}
json.dump(bundle_contents, s)
s.seek(0)
s = six.BytesIO(s.getvalue().encode('ascii'))
m.get(TestArgs.murano_repo_url + '/bundles/test_bundle.bundle',
body=s)
@ -966,14 +965,14 @@ class ShellPackagesOperations(ShellCommandTest):
m.get(TestArgs.murano_repo_url + '/apps/first_app.zip', body=pkg1)
m.get(TestArgs.murano_repo_url + '/apps/second_app.1.0.zip',
body=pkg2)
s = StringIO.StringIO()
s = six.StringIO()
# bundle only contains 1st package
bundle_contents = {'Packages': [
{'Name': 'first_app'},
]}
json.dump(bundle_contents, s)
s.seek(0)
s = six.BytesIO(s.getvalue().encode('ascii'))
m.get(TestArgs.murano_repo_url + '/bundles/test_bundle.bundle',
body=s)
@ -999,13 +998,13 @@ class ShellPackagesOperations(ShellCommandTest):
m.get(TestArgs.murano_repo_url + '/apps/first_app.zip', body=pkg1)
m.get(TestArgs.murano_repo_url + '/apps/second_app.1.0.zip',
body=pkg2)
s = StringIO.StringIO()
s = six.StringIO()
bundle_contents = {'Packages': [
{'Name': 'first_app'},
{'Name': 'second_app', 'Version': '1.0'}
]}
json.dump(bundle_contents, s)
s.seek(0)
s = six.BytesIO(s.getvalue().encode('ascii'))
url = 'http://127.0.0.2/test_bundle.bundle'
m.get(url, body=s)
@ -1061,9 +1060,9 @@ class ShellPackagesOperations(ShellCommandTest):
'Require': {'third_app': None}})
pkg2 = make_pkg({'FullName': 'second_app'})
pkg3 = make_pkg({'FullName': 'third_app'})
with open(os.path.join(tmp_dir, 'first_app'), 'w') as f:
with open(os.path.join(tmp_dir, 'first_app'), 'wb') as f:
f.write(pkg1.read())
with open(os.path.join(tmp_dir, 'third_app'), 'w') as f:
with open(os.path.join(tmp_dir, 'third_app'), 'wb') as f:
f.write(pkg3.read())
m.get(TestArgs.murano_repo_url + '/apps/first_app.zip',
@ -1098,12 +1097,12 @@ class ShellPackagesOperations(ShellCommandTest):
m.get(TestArgs.murano_repo_url + '/apps/test_app.zip', body=pkg)
s = StringIO.StringIO()
s = six.StringIO()
expected_bundle = {'Packages': [
{'Name': 'test_app'},
]}
json.dump(expected_bundle, s)
s.seek(0)
s = six.BytesIO(s.getvalue().encode('ascii'))
m.get(TestArgs.murano_repo_url + '/bundles/test_bundle.bundle',
body=s)

15
muranoclient/tests/unit/test_utils.py
View File

@ -15,17 +15,16 @@
import json
import os.path
import StringIO
import tempfile
import zipfile
import mock
import requests
import requests_mock
import six
import testtools
import yaml
from muranoclient.common import utils
@ -46,7 +45,7 @@ class FileTest(testtools.TestCase):
def test_file_object_url_fails(self):
resp = requests.Response()
resp.status_code = 400
resp.raw = StringIO.StringIO("123")
resp.raw = six.BytesIO(six.b("123"))
with mock.patch(
'requests.get',
@ -56,7 +55,7 @@ class FileTest(testtools.TestCase):
def test_file_object_url(self):
resp = requests.Response()
resp.raw = StringIO.StringIO("123")
resp.raw = six.BytesIO(six.b("123"))
resp.status_code = 200
with mock.patch(
'requests.get',
@ -75,7 +74,7 @@ def make_pkg(manifest_override, image_dicts=None):
'Name': 'Apache HTTP Server',
'Type': 'Application'}
manifest.update(manifest_override)
file_obj = StringIO.StringIO()
file_obj = six.BytesIO()
zfile = zipfile.ZipFile(file_obj, "a")
zfile.writestr('manifest.yaml', yaml.dump(manifest))
if image_dicts:
@ -212,7 +211,7 @@ class PackageTest(testtools.TestCase):
def test_file_object_repo_fails(self):
resp = requests.Response()
resp.raw = StringIO.StringIO("123")
resp.raw = six.BytesIO(six.b("123"))
resp.status_code = 400
with mock.patch(
'requests.get',
@ -227,7 +226,7 @@ class PackageTest(testtools.TestCase):
def test_file_object_repo(self):
resp = requests.Response()
resp.raw = StringIO.StringIO("123")
resp.raw = six.BytesIO(six.b("123"))
resp.status_code = 200
with mock.patch(
'requests.get',
@ -242,7 +241,7 @@ class BundleTest(testtools.TestCase):
@requests_mock.mock()
def test_packages(self, m):
s = StringIO.StringIO()
s = six.StringIO()
bundle_contents = {'Packages': [
{'Name': 'first_app'},
{'Name': 'second_app', 'Version': '1.0'}

7
muranoclient/v1/environments.py
View File

@ -11,7 +11,7 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import urllib
from six.moves import urllib
from muranoclient.common import base
@ -37,7 +37,7 @@ class EnvironmentManager(base.ManagerWithFind):
def list(self, all_tenants=False):
path = '/v1/environments?{query}'.format(
query=urllib.urlencode({'all_tenants': all_tenants}))
query=urllib.parse.urlencode({'all_tenants': all_tenants}))
return self._list(path, 'environments')
def create(self, data):
@ -49,7 +49,8 @@ class EnvironmentManager(base.ManagerWithFind):
def delete(self, environment_id, abandon=False):
path = '/v1/environments/{id}?{query}'.format(
id=environment_id, query=urllib.urlencode({'abandon': abandon}))
id=environment_id,
query=urllib.parse.urlencode({'abandon': abandon}))
return self._delete(path)
def get(self, environment_id, session_id=None):

2
muranoclient/v1/package_creator/hot_package.py
View File

@ -42,7 +42,7 @@ def generate_manifest(args):
normalized_name = normalized_name.title().replace(' ', '')
args.full_name = '{0}.{1}'.format(prefix, normalized_name)
try:
with open(args.template) as heat_file:
with open(args.template, 'rb') as heat_file:
yaml_content = yaml.load(heat_file)
if not args.description:
args.description = yaml_content.get(

12
muranoclient/v1/packages.py
View File

@ -13,7 +13,7 @@
# under the License.
import six
import urllib
from six.moves import urllib
from oslo_serialization import jsonutils
import yaml
@ -75,7 +75,8 @@ class PackageManager(base.Manager):
v = v.encode('utf-8')
params[k] = v
return '?'.join(
['/v1/catalog/packages', urllib.urlencode(params, doseq=True)]
['/v1/catalog/packages',
urllib.parse.urlencode(params, doseq=True)]
)
def paginate(_url):
@ -84,10 +85,9 @@ class PackageManager(base.Manager):
for image in body['packages']:
yield image
try:
next_url = construct_url(
dict(kwargs.items() +
{'marker': body['next_marker']}.items())
)
m_kwargs = kwargs.copy()
m_kwargs['marker'] = body['next_marker']
next_url = construct_url(m_kwargs)
except KeyError:
return
else:

12
muranoclient/v1/shell.py
View File

@ -16,6 +16,8 @@ import functools
import json
import os
import shutil
import six
import six.moves
import sys
import tempfile
import uuid
@ -466,7 +468,7 @@ def _handle_package_exists(mc, data, package, exists_action):
if not res:
while True:
print("What do you want to do? (s)kip, (u)pdate, (a)bort")
res = raw_input()
res = six.moves.input()
if res in allowed_results:
break
if res == 's':
@ -552,7 +554,7 @@ def do_package_import(mc, args):
should_do_list = True
total_reqs.update(package.requirements(base_url=args.murano_repo_url))
for name, package in total_reqs.iteritems():
for name, package in six.iteritems(total_reqs):
image_specs = package.images()
if image_specs:
print("Inspecting required images")
@ -654,7 +656,7 @@ def do_bundle_import(mc, args):
)
total_reqs.update(requirements)
for name, dep_package in total_reqs.iteritems():
for name, dep_package in six.iteritems(total_reqs):
image_specs = dep_package.images()
if image_specs:
print("Inspecting required images")
@ -686,7 +688,7 @@ def do_bundle_import(mc, args):
def _handle_save_packages(packages, dst, base_url, no_images):
downloaded_images = []
for name, pkg in packages.iteritems():
for name, pkg in six.iteritems(packages):
if not no_images:
image_specs = pkg.images()
for image_spec in image_specs:
@ -763,7 +765,7 @@ def do_bundle_save(mc, args):
_handle_save_packages(total_reqs, dst, base_url, no_images)
try:
bundle_file.save(dst)
bundle_file.save(dst, binary=False)
print("Bundle file {0} has been successfully saved".format(bundle))
except Exception as e:
print("Error {0} occurred while saving bundle {1}".format(e, bundle))

2
tox.ini
View File

@ -1,5 +1,5 @@
[tox]
envlist = py26,py27,pep8
envlist = py26,py27,pep8,py34
minversion = 1.6
skipsdist = True