diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 074674e7..ba680ca5 100755 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -58,6 +58,26 @@ function mkdir_chown_stack { } +function configure_k8s_certificates { + pushd $QINLING_DIR + mkdir_chown_stack "$QINLING_CONF_DIR"/pki/kubernetes + + curl -L https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -o /tmp/cfssl + chmod +x /tmp/cfssl + curl -L https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -o /tmp/cfssljson + chmod +x /tmp/cfssljson + + sudo /tmp/cfssl gencert -ca=/etc/kubernetes/pki/ca.crt -ca-key=/etc/kubernetes/pki/ca.key -config=example/kubernetes/cfssl-ca-config.json -profile=client example/kubernetes/cfssl-client-csr.json | /tmp/cfssljson -bare client + # The command above outputs client-key.pem and client.pem + mv client-key.pem "$QINLING_CONF_DIR"/pki/kubernetes/qinling.key + mv client.pem "$QINLING_CONF_DIR"/pki/kubernetes/qinling.crt + + cp /etc/kubernetes/pki/ca.crt "$QINLING_CONF_DIR"/pki/kubernetes/ca.crt + + popd +} + + function configure_qinling { mkdir_chown_stack "$QINLING_AUTH_CACHE_DIR" rm -f "$QINLING_AUTH_CACHE_DIR"/* @@ -89,6 +109,15 @@ function configure_qinling { # Configure the database. iniset $QINLING_CONF_FILE database connection `database_connection_url qinling` + + # Configure Kubernetes API server certificates for qinling if required. + if [ "$QINLING_K8S_APISERVER_TLS" == "True" ]; then + iniset $QINLING_CONF_FILE kubernetes kube_host https://$(hostname -f):6443 + configure_k8s_certificates + sudo kubectl create -f $QINLING_DIR/example/kubernetes/k8s_qinling_role.yaml + else + iniset $QINLING_CONF_FILE kubernetes use_api_certificate False + fi } diff --git a/devstack/settings b/devstack/settings index d3057a2d..8bc33604 100644 --- a/devstack/settings +++ b/devstack/settings @@ -24,3 +24,5 @@ QINLING_AUTH_CACHE_DIR=${QINLING_AUTH_CACHE_DIR:-/var/cache/qinling} QINLING_FUNCTION_STORAGE_DIR=${QINLING_FUNCTION_STORAGE_DIR:-/opt/qinling/funtion/packages} QINLING_PYTHON_RUNTIME_IMAGE=${QINLING_PYTHON_RUNTIME_IMAGE:-openstackqinling/python-runtime} QINLING_NODEJS_RUNTIME_IMAGE=${QINLING_NODEJS_RUNTIME_IMAGE:-openstackqinling/nodejs-runtime} + +QINLING_K8S_APISERVER_TLS=${QINLING_K8S_APISERVER_TLS:-True} diff --git a/example/kubernetes/cfssl-ca-config.json b/example/kubernetes/cfssl-ca-config.json new file mode 100644 index 00000000..c5fc98be --- /dev/null +++ b/example/kubernetes/cfssl-ca-config.json @@ -0,0 +1,17 @@ +{ + "signing": { + "default": { + "expiry": "168h" + }, + "profiles": { + "client": { + "expiry": "8760h", + "usages": [ + "signing", + "key encipherment", + "client auth" + ] + } + } + } +} diff --git a/example/kubernetes/cfssl-client-csr.json b/example/kubernetes/cfssl-client-csr.json new file mode 100644 index 00000000..6626637d --- /dev/null +++ b/example/kubernetes/cfssl-client-csr.json @@ -0,0 +1,7 @@ +{ + "CN": "qinling", + "key": { + "algo": "rsa", + "size": 2048 + } +} diff --git a/example/kubernetes/k8s_qinling_role.yaml b/example/kubernetes/k8s_qinling_role.yaml new file mode 100644 index 00000000..99887df8 --- /dev/null +++ b/example/kubernetes/k8s_qinling_role.yaml @@ -0,0 +1,74 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: qinling +rules: + - apiGroups: [""] + resources: ["nodes", "namespaces"] + verbs: ["list"] + - apiGroups: [""] + resources: ["namespaces"] + resourceNames: ["qinling"] + verbs: ["create"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: qinling +subjects: +- kind: User + name: qinling + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: ClusterRole + name: qinling + apiGroup: rbac.authorization.k8s.io +--- +# The qinling namespace should be created for the role and rolebinding +apiVersion: v1 +kind: Namespace +metadata: + name: qinling +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: qinling + namespace: qinling +rules: +- apiGroups: [""] + resources: ["services"] + verbs: ["list", "get", "create", "delete"] +- apiGroups: [""] + resources: ["pods"] + verbs: ["list", "get", "create", "patch", "delete", "deletecollection"] +- apiGroups: [""] + resources: ["pods/log"] + verbs: ["get"] +- apiGroups: ["extensions"] + resources: ["deployments"] + verbs: ["get", "create", "patch", "deletecollection"] +- apiGroups: ["extensions"] + resources: ["deployments/rollback"] + verbs: ["create"] +- apiGroups: ["extensions"] + resources: ["deployments/status"] + verbs: ["get"] +- apiGroups: ["extensions"] + resources: ["replicasets"] + verbs: ["deletecollection"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: qinling + namespace: qinling +subjects: +- kind: User + name: qinling + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: Role + name: qinling + apiGroup: rbac.authorization.k8s.io diff --git a/qinling/config.py b/qinling/config.py index 3b318d89..d716a07c 100644 --- a/qinling/config.py +++ b/qinling/config.py @@ -146,6 +146,30 @@ kubernetes_opts = [ help='Kubernetes server address, e.g. you can start a proxy to the ' 'Kubernetes API server by using "kubectl proxy" command.' ), + cfg.BoolOpt( + 'use_api_certificate', + default=True, + help='Whether to use client certificates to connect to the ' + 'Kubernetes API server.' + ), + cfg.StrOpt( + 'ssl_ca_cert', + default='/etc/qinling/pki/kubernetes/ca.crt', + help='Path to the CA certificate for qinling to use to connect to ' + 'the Kubernetes API server.' + ), + cfg.StrOpt( + 'cert_file', + default='/etc/qinling/pki/kubernetes/qinling.crt', + help='Path to the client certificate for qinling to use to ' + 'connect to the Kubernetes API server.' + ), + cfg.StrOpt( + 'key_file', + default='/etc/qinling/pki/kubernetes/qinling.key', + help='Path to the client certificate key file for qinling to use to ' + 'connect to the Kubernetes API server.' + ), cfg.StrOpt( 'log_devel', default='INFO', diff --git a/qinling/orchestrator/kubernetes/utils.py b/qinling/orchestrator/kubernetes/utils.py index f8ef3c1b..793d79a4 100644 --- a/qinling/orchestrator/kubernetes/utils.py +++ b/qinling/orchestrator/kubernetes/utils.py @@ -22,7 +22,12 @@ from kubernetes.client import configuration as k8s_config def get_k8s_clients(conf): config = k8s_config.Configuration() config.host = conf.kubernetes.kube_host - config.verify_ssl = False + if conf.kubernetes.use_api_certificate: + config.ssl_ca_cert = conf.kubernetes.ssl_ca_cert + config.cert_file = conf.kubernetes.cert_file + config.key_file = conf.kubernetes.key_file + else: + config.verify_ssl = False client = api_client.ApiClient(configuration=config) v1 = core_v1_api.CoreV1Api(client) v1extention = extensions_v1beta1_api.ExtensionsV1beta1Api(client) diff --git a/qinling_tempest_plugin/config.py b/qinling_tempest_plugin/config.py index fae32221..906bd354 100644 --- a/qinling_tempest_plugin/config.py +++ b/qinling_tempest_plugin/config.py @@ -40,7 +40,4 @@ QinlingGroup = [ choices=['public', 'admin', 'internal', 'publicURL', 'adminURL', 'internalURL'], help="The endpoint type to use for the qinling service."), - cfg.StrOpt('kube_host', - default='http://127.0.0.1:8001', - help="The Kubernetes service address."), ] diff --git a/qinling_tempest_plugin/tests/api/test_runtimes.py b/qinling_tempest_plugin/tests/api/test_runtimes.py index d76d2a9f..aca0f7d8 100644 --- a/qinling_tempest_plugin/tests/api/test_runtimes.py +++ b/qinling_tempest_plugin/tests/api/test_runtimes.py @@ -43,19 +43,10 @@ class RuntimesTest(base.BaseQinlingTest): ) # Wait for runtime to be available + # We don't have to check k8s resource, if runtime's status has changed + # to available, then kubernetes deployment is assumed to be ok. self.await_runtime_available(runtime_id) - # Check k8s resource - deploy = self.k8s_v1extention.read_namespaced_deployment( - runtime_id, - namespace=self.namespace - ) - - self.assertEqual(runtime_id, deploy.metadata.name) - self.assertEqual( - deploy.status.replicas, deploy.status.available_replicas - ) - # Delete runtime resp = self.admin_client.delete_resource('runtimes', runtime_id) diff --git a/qinling_tempest_plugin/tests/base.py b/qinling_tempest_plugin/tests/base.py index 04a2781c..5daf2dc5 100644 --- a/qinling_tempest_plugin/tests/base.py +++ b/qinling_tempest_plugin/tests/base.py @@ -21,8 +21,6 @@ from tempest.lib.common.utils import data_utils from tempest import test import tenacity -from qinling_tempest_plugin.tests import utils - CONF = config.CONF @@ -46,13 +44,6 @@ class BaseQinlingTest(test.BaseTestCase): cls.alt_client = cls.os_alt.qinling.QinlingClient() cls.admin_client = cls.os_admin.qinling.QinlingClient() - # Initilize k8s client - clients = utils.get_k8s_clients(CONF) - cls.k8s_v1 = clients['v1'] - cls.k8s_v1extention = clients['v1extention'] - # cls.k8s_apps_v1 = clients['apps_v1'] - cls.namespace = 'qinling' - @classmethod def resource_setup(cls): super(BaseQinlingTest, cls).resource_setup() diff --git a/qinling_tempest_plugin/tests/utils.py b/qinling_tempest_plugin/tests/utils.py index 8a068869..a2acc8ad 100644 --- a/qinling_tempest_plugin/tests/utils.py +++ b/qinling_tempest_plugin/tests/utils.py @@ -14,30 +14,6 @@ import hashlib -from kubernetes.client import api_client -# from kubernetes.client.apis import apps_v1_api -from kubernetes.client.apis import core_v1_api -from kubernetes.client.apis import extensions_v1beta1_api -from kubernetes.client import configuration as k8s_config - - -def get_k8s_clients(conf): - config = k8s_config.Configuration() - config.host = conf.qinling.kube_host - config.verify_ssl = False - client = api_client.ApiClient(configuration=config) - v1 = core_v1_api.CoreV1Api(client) - v1extention = extensions_v1beta1_api.ExtensionsV1beta1Api(client) - # apps_v1 = apps_v1_api.AppsV1Api(client) - - clients = { - 'v1': v1, - 'v1extention': v1extention - # 'apps_v1': apps_v1 - } - - return clients - def md5(file=None, content=None): hash_md5 = hashlib.md5() diff --git a/tools/gate/kubeadm/setup_gate.sh b/tools/gate/kubeadm/setup_gate.sh index a24ed80d..2f8b37af 100755 --- a/tools/gate/kubeadm/setup_gate.sh +++ b/tools/gate/kubeadm/setup_gate.sh @@ -40,7 +40,10 @@ while true; do [ $now -gt $end ] && echo "Failed to setup kubernetes cluster in time" && exit -1 done -create_k8s_screen +if [ "$QINLING_K8S_APISERVER_TLS" != "True" ]; then + # Kubernetes proxy is needed if we don't use secure connections. + create_k8s_screen +fi #net_hosts_post_kube #net_resolv_post_kube