K8s and Neutron clients support

Adds basic K8s client implementation and CONF-based singletons for
both Neutron and K8s clients.

The K8s client added by this patch should be considered a temporary
solution that only implements the necessary parts to let us move
forward with kuryr-kubernetes. Eventually it will be replaced by either
[1] or [2].

The problem with [1] is that it does not yet support the streaming API
that we need for WATCH. And [2] is outside of the OSt umbrella, so [1]
is preferred over [2] unless [2] makes it into global-requirements.txt.

[1] https://github.com/openstack/python-k8sclient
[2] https://pypi.python.org/pypi/pykube

NOTE: Removed py3-related code from config and top-level __init__.
      How to properly deal with that code is TBD.

Change-Id: Ib4eb410eaf9725c296fcdddd8857eb24b8929915
Partially-Implements: blueprint kuryr-k8s-integration

kuryr_kubernetes/__init__.py

@@ -12,13 +12,6 @@
 
 import pbr.version
 
-from kuryr_kubernetes.translators import port
-from kuryr_kubernetes import watchers
-
 
 __version__ = pbr.version.VersionInfo(
     'kuryr_kubernetes').version_string()
-
-CONFIG_MAP = {
-    watchers.PodWatcher: [port.PortTranslator]
-}

kuryr_kubernetes/clients.py

@@ -0,0 +1,37 @@
+# Copyright (c) 2016 Mirantis, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from kuryr.lib import utils
+
+from kuryr_kubernetes import config
+from kuryr_kubernetes import k8s_client
+
+_clients = {}
+_NEUTRON_CLIENT = 'neutron-client'
+_KUBERNETES_CLIENT = 'kubernetes-client'
+
+
+def get_neutron_client():
+    return _clients[_NEUTRON_CLIENT]
+
+
+def get_kubernetes_client():
+    return _clients[_KUBERNETES_CLIENT]
+
+
+def setup_clients():
+    _clients[_NEUTRON_CLIENT] = utils.get_neutron_client()
+    _clients[_KUBERNETES_CLIENT] = k8s_client.K8sClient(
+        config.CONF.kubernetes.api_root)

kuryr_kubernetes/config.py

@@ -33,10 +33,6 @@ k8s_opts = [
     cfg.StrOpt('api_root',
         help=_("The root URL of the Kubernetes API"),
         default=os.environ.get('K8S_API', 'http://localhost:8080')),
-    cfg.StrOpt('config_map',
-        help=_("The dict that stores the relationships between watchers and"
-               " translators."),
-        default=('kuryr_kubernetes.CONFIG_MAP'))
 ]
 
 

kuryr_kubernetes/exceptions.py

@@ -0,0 +1,18 @@
+# Copyright (c) 2016 Mirantis, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+class K8sClientException(Exception):
+    pass

kuryr_kubernetes/k8s_client.py

@@ -0,0 +1,66 @@
+# Copyright (c) 2016 Mirantis, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import contextlib
+
+from oslo_serialization import jsonutils
+import requests
+
+from kuryr_kubernetes import exceptions as exc
+
+
+class K8sClient(object):
+    # REVISIT(ivc): replace with python-k8sclient if it could be extended
+    # with 'WATCH' support
+
+    def __init__(self, base_url):
+        self._base_url = base_url
+
+    def get(self, path):
+        url = self._base_url + path
+        response = requests.get(url)
+        if not response.ok:
+            raise exc.K8sClientException(response.text)
+        return response.json()
+
+    def annotate(self, path, annotations):
+        url = self._base_url + path
+        data = jsonutils.dumps({
+            "metadata": {
+                "annotations": annotations
+            }
+        })
+        response = requests.patch(url, data=data, headers={
+            'Content-Type': 'application/merge-patch+json',
+            'Accept': 'application/json',
+        })
+        if not response.ok:
+            raise exc.K8sClientException(response.text)
+        return response.json()['metadata']['annotations']
+
+    def watch(self, path):
+        params = {'watch': 'true'}
+        url = self._base_url + path
+
+        # TODO(ivc): handle connection errors and retry on failure
+        while True:
+            with contextlib.closing(requests.get(url, params=params,
+                                                 stream=True)) as response:
+                if not response.ok:
+                    raise exc.K8sClientException(response.text)
+                for line in response.iter_lines(delimiter='\n'):
+                    line = line.strip()
+                    if line:
+                        yield jsonutils.loads(line)

kuryr_kubernetes/tests/unit/test_clients.py

@@ -0,0 +1,41 @@
+# Copyright (c) 2016 Mirantis, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+
+from kuryr_kubernetes import clients
+from kuryr_kubernetes.tests import base as test_base
+
+
+class TestK8sClient(test_base.TestCase):
+
+    @mock.patch('kuryr_kubernetes.config.CONF')
+    @mock.patch('kuryr_kubernetes.k8s_client.K8sClient')
+    @mock.patch('kuryr.lib.utils.get_neutron_client')
+    def test_setup_clients(self, m_neutron, m_k8s, m_cfg):
+        k8s_api_root = 'http://127.0.0.1:1234'
+
+        neutron_dummy = object()
+        k8s_dummy = object()
+
+        m_cfg.kubernetes.api_root = k8s_api_root
+        m_neutron.return_value = neutron_dummy
+        m_k8s.return_value = k8s_dummy
+
+        clients.setup_clients()
+
+        m_k8s.assert_called_with(k8s_api_root)
+        self.assertIs(k8s_dummy, clients.get_kubernetes_client())
+        self.assertIs(neutron_dummy, clients.get_neutron_client())

kuryr_kubernetes/tests/unit/test_k8s_client.py

@@ -0,0 +1,113 @@
+# Copyright (c) 2016 Mirantis, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import itertools
+import mock
+
+from oslo_serialization import jsonutils
+
+from kuryr_kubernetes import exceptions as exc
+from kuryr_kubernetes import k8s_client
+from kuryr_kubernetes.tests import base as test_base
+
+
+class TestK8sClient(test_base.TestCase):
+    def setUp(self):
+        super(TestK8sClient, self).setUp()
+        self.base_url = 'http://127.0.0.1:12345'
+        self.client = k8s_client.K8sClient(self.base_url)
+
+    @mock.patch('requests.get')
+    def test_get(self, m_get):
+        path = '/test'
+        ret = {'test': 'value'}
+
+        m_resp = mock.MagicMock()
+        m_resp.ok = True
+        m_resp.json.return_value = ret
+        m_get.return_value = m_resp
+
+        self.assertEqual(ret, self.client.get(path))
+        m_get.assert_called_once_with(self.base_url + path)
+
+    @mock.patch('requests.get')
+    def test_get_exception(self, m_get):
+        path = '/test'
+
+        m_resp = mock.MagicMock()
+        m_resp.ok = False
+        m_get.return_value = m_resp
+
+        self.assertRaises(exc.K8sClientException, self.client.get, path)
+
+    @mock.patch('requests.patch')
+    def test_annotate(self, m_patch):
+        path = '/test'
+        annotations = {'a1': 'v1', 'a2': 'v2'}
+        ret = {'metadata': {'annotations': annotations}}
+        data = jsonutils.dumps(ret, sort_keys=True)
+
+        m_resp = mock.MagicMock()
+        m_resp.ok = True
+        m_resp.json.return_value = ret
+        m_patch.return_value = m_resp
+
+        self.assertEqual(annotations, self.client.annotate(path, annotations))
+        m_patch.assert_called_once_with(self.base_url + path,
+                                        data=data, headers=mock.ANY)
+
+    @mock.patch('requests.patch')
+    def test_annotate_exception(self, m_patch):
+        path = '/test'
+
+        m_resp = mock.MagicMock()
+        m_resp.ok = False
+        m_patch.return_value = m_resp
+
+        self.assertRaises(exc.K8sClientException, self.client.annotate,
+                          path, {})
+
+    @mock.patch('requests.get')
+    def test_watch(self, m_get):
+        path = '/test'
+        data = [{'obj': 'obj%s' % i} for i in range(3)]
+        lines = [jsonutils.dumps(i) for i in data]
+
+        m_resp = mock.MagicMock()
+        m_resp.ok = True
+        m_resp.iter_lines.return_value = lines
+        m_get.return_value = m_resp
+
+        cycles = 3
+        self.assertEqual(
+            data * cycles,
+            list(itertools.islice(self.client.watch(path),
+                                  len(data) * cycles)))
+
+        self.assertEqual(cycles, m_get.call_count)
+        self.assertEqual(cycles, m_resp.close.call_count)
+        m_get.assert_called_with(self.base_url + path, stream=True,
+                                 params={'watch': 'true'})
+
+    @mock.patch('requests.get')
+    def test_watch_exception(self, m_get):
+        path = '/test'
+
+        m_resp = mock.MagicMock()
+        m_resp.ok = False
+        m_get.return_value = m_resp
+
+        self.assertRaises(exc.K8sClientException, next,
+                          self.client.watch(path))

requirements.txt

@@ -3,3 +3,11 @@
 # process, which may cause wedges in the gate later.
 
 -e git://github.com/openstack/kuryr.git@master#egg=kuryr_lib
+pbr>=1.6 # Apache-2.0
+requests>=2.10.0 # Apache-2.0
+oslo.config>=3.14.0 # Apache-2.0
+oslo.i18n>=2.1.0 # Apache-2.0
+oslo.log>=3.11.0 # Apache-2.0
+oslo.serialization>=1.10.0 # Apache-2.0
+oslo.utils>=3.16.0 # Apache-2.0
+six>=1.9.0 # MIT

test-requirements.txt

@@ -5,6 +5,7 @@
 hacking<0.12,>=0.10.2 # Apache-2.0
 
 coverage>=3.6 # Apache-2.0
+mock>=2.0 # BSD
 ddt>=1.0.1 # MIT
 python-subunit>=0.0.18 # Apache-2.0/BSD
 sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2 # BSD

tox.ini

@@ -1,6 +1,6 @@
 [tox]
 minversion = 1.6
-envlist = py35,pep8
+envlist = py27,py35,pep8
 skipsdist = True
 
 [testenv]
@@ -14,11 +14,8 @@ deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
 whitelist_externals = sh
                       find
-commands = find . -type f -name "*.py[c|o]" -delete
-           sh tools/pretty_tox.sh '{posargs}'
-
-[testenv:py27]
-commands =
+commands = find {toxinidir} -type f -name "*.py[c|o]" -delete
+           sh {toxinidir}/tools/pretty_tox.sh '{posargs}'
 
 [testenv:fullstack]
 basepython = python2.7