diff --git a/doc/source/test_plans/kubernetes_density/plan.rst b/doc/source/test_plans/kubernetes_density/plan.rst index 779ae5e..a959e9a 100644 --- a/doc/source/test_plans/kubernetes_density/plan.rst +++ b/doc/source/test_plans/kubernetes_density/plan.rst @@ -65,6 +65,34 @@ List of performance metrics +-------------------------+---------------------------------------------+ +Test Case #2: Measure Kubelet capacity +-------------------------------------- + +Description +^^^^^^^^^^^ +The goal of this test is to investigate Kubelet capacity. For this, rerun +test case #1 for different number of nodes. + +List of performance metrics +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. table:: list of test metrics to be collected during this test + + +-------------------------+---------------------------------------------+ + | Parameter | Description | + +=========================+=============================================+ + | POD_COUNT | Number of pods | + +-------------------------+---------------------------------------------+ + | NODE_COUNT | Number of nodes | + +-------------------------+---------------------------------------------+ + | POD_FIRST_REPORT | Time taken by pod to start and report | + +-------------------------+---------------------------------------------+ + | KUBECTL_RUN | Time for all pods to be reported as running | + +-------------------------+---------------------------------------------+ + | KUBECTL_TERMINATE | Time to terminate all pods | + +-------------------------+---------------------------------------------+ + + Reports ======= diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/100x100.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/100x100.png new file mode 100644 index 0000000..0d72150 Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/100x100.png differ diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/100x100_term.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/100x100_term.png new file mode 100644 index 0000000..dfc1c63 Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/100x100_term.png differ diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/100x200.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/100x200.png new file mode 100644 index 0000000..05e7404 Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/100x200.png differ diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/100x200_term.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/100x200_term.png new file mode 100644 index 0000000..9c76bb3 Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/100x200_term.png differ diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/100x50.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/100x50.png new file mode 100644 index 0000000..65a022a Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/100x50.png differ diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/100x50_term.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/100x50_term.png new file mode 100644 index 0000000..8550d50 Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/100x50_term.png differ diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/200x100.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/200x100.png new file mode 100644 index 0000000..9cecae4 Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/200x100.png differ diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/200x100_term.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/200x100_term.png new file mode 100644 index 0000000..c387dda Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/200x100_term.png differ diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/200x200.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/200x200.png new file mode 100644 index 0000000..1723a85 Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/200x200.png differ diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/200x200_term.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/200x200_term.png new file mode 100644 index 0000000..9b608d5 Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/200x200_term.png differ diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/200x50.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/200x50.png new file mode 100644 index 0000000..3ea24cf Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/200x50.png differ diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/200x50_term.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/200x50_term.png new file mode 100644 index 0000000..8d1911f Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/200x50_term.png differ diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/50x100.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/50x100.png new file mode 100644 index 0000000..feb527b Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/50x100.png differ diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/50x100_term.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/50x100_term.png new file mode 100644 index 0000000..1ebb04c Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/50x100_term.png differ diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/50x200.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/50x200.png new file mode 100644 index 0000000..5ef8e81 Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/50x200.png differ diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/50x200_term.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/50x200_term.png new file mode 100644 index 0000000..da51b8c Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/50x200_term.png differ diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/50x400.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/50x400.png new file mode 100644 index 0000000..8aa83ea Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/50x400.png differ diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/50x400_term.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/50x400_term.png new file mode 100644 index 0000000..7f1d7ec Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/50x400_term.png differ diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/50x50.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/50x50.png new file mode 100644 index 0000000..340c39a Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/50x50.png differ diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/50x50_term.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/50x50_term.png new file mode 100644 index 0000000..051e8a2 Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/50x50_term.png differ diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/data/reports.tar.bz2 b/doc/source/test_results/container_cluster_systems/kubernetes/density/data/reports.tar.bz2 new file mode 100644 index 0000000..12899fc Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/data/reports.tar.bz2 differ diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/index.rst b/doc/source/test_results/container_cluster_systems/kubernetes/density/index.rst index 5cad40d..8316b49 100644 --- a/doc/source/test_results/container_cluster_systems/kubernetes/density/index.rst +++ b/doc/source/test_results/container_cluster_systems/kubernetes/density/index.rst @@ -27,12 +27,18 @@ Node roles: - node3: minion+etcd - node4: minion -Software versions: +Software versions (test case #1): - OS: Ubuntu 16.04.1 LTS (Xenial Xerus) - Kernel: 4.4.0-47 - Docker: 1.12.1 - Kubernetes: 1.4.3 +Software versions (test case #2): + - OS: Ubuntu 16.04.1 LTS (Xenial Xerus) + - Kernel: 4.4.0-36 + - Docker: 1.13.1 + - Kubernetes: 1.5.3 + Reports ======= @@ -176,6 +182,252 @@ System metrics from API nodes and minion are below Full `Kubernetes stats`_ are available online. +Test Case #2: Measure Kubelet capacity +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Pod startup time is measured with help of +`MMM(MySQL/Master/Minions) testing suite`_. Original code was updated. We added +automatic creation of charts with pod's status, when pod startup (or down). To +schedule all pods on a single node the original replication controller for +minions is updated with scheduler hint. To do this add the following lines +into template's spec section: + +.. code-block:: yaml + + nodeSelector: + kubernetes.io/hostname: + +Pod status from Kubernetes point of view is retrieved from kubectl tool. +The process is automated with +:download:`kubectl_mon_v2.py `, which collects +information about pod's status and sends to database. Charts are created by +updated `MMM(MySQL/Master/Minions) testing suite`_. + +Every measurement starts with empty namespace. Then Kubernetes replication +controller is created with specified number of pods. We collect pod's report +time and kubectl stats. The summary data is presented below. + +.. list-table:: POD stats + :header-rows: 1 + + * + - POD_COUNT + - NODE_COUNT + - POD_FIRST_REPORT, s + - KUBECTL_RUN, s + - KUBECTL_TERMINATE, s + * + - 50 + - 50 + - 197 + - 290 + - 289 + * + - 100 + - 50 + - 415 + - 597 + - 577 + * + - 200 + - 50 + - 952 + - 1218 + - 1154 + * + - 50 + - 100 + - 381 + - 425 + - 536 + * + - 100 + - 100 + - 788 + - 2093 (with errors) + - 1076 + * + - 200 + - 100 + - 2653 + - 3838 (not finished) + - 3001 (not finished) + * + - 50 + - 200 + - 970 + - 1256 + - 1032 + * + - 100 + - 200 + - 1632 + - 3225 + - 2248 + * + - 200 + - 200 + - 7098 (6.5% lost) + - 8075 (not finished) + - ∞ (not finished) + * + - 50 + - 400 + - 1823 + - 2667 (with errors) + - 2038 + * + - 100 + - 400 + - 7582 + - 8262 (with errors) + - 5200 + * + - 400 + - 50 + - 9607 + - ∞ (not finished) + - ∞ (not finished) + + +Detailed Stats +-------------- + +Note: You can download these reports in HTML format +:download:`here ` + +50 pods (~1 pod per core) on 50 nodes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Start replication controller with 50 pods on 50 nodes + +.. image:: 50x50.png + :width: 100% + +Terminate replication controller with 50 pods on 50 nodes + +.. image:: 50x50_term.png + :width: 100% + +100 pods (~2 pod per core) on 50 nodes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Start replication controller with 100 pods on 50 nodes + +.. image:: 50x100.png + :width: 100% + +Terminate replication controller with 100 pods on 50 nodes + +.. image:: 50x100_term.png + :width: 100% + +200 pods (~4 pod per core) on 50 nodes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Start replication controller with 200 pods on 50 nodes + +.. image:: 50x200.png + :width: 100% + +Terminate replication controller with 200 pods on 50 nodes + +.. image:: 50x200_term.png + :width: 100% + +50 pods (~1 pod per core) on 100 nodes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Start replication controller with 50 pods on 100 nodes + +.. image:: 100x50.png + :width: 100% + +Terminate replication controller with 50 pods on 100 nodes + +.. image:: 100x50_term.png + :width: 100% + +100 pods (~2 pod per core) on 100 nodes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Start replication controller with 100 pods on 100 nodes + +.. image:: 100x100.png + :width: 100% + +Terminate replication controller with 100 pods on 100 nodes + +.. image:: 100x100_term.png + :width: 100% + +200 pods (~4 pod per core) on 100 nodes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Start replication controller with 200 pods on 100 nodes + +.. image:: 100x200.png + :width: 100% + +Terminate replication controller with 200 pods on 100 nodes + +.. image:: 100x200_term.png + :width: 100% + +50 pods (~1 pod per core) on 200 nodes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Start replication controller with 50 pods on 100 nodes + +.. image:: 200x50.png + :width: 100% + +Terminate replication controller with 50 pods on 100 nodes + +.. image:: 200x50_term.png + :width: 100% + +100 pods (~2 pod per core) on 200 nodes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Start replication controller with 100 pods on 200 nodes + +.. image:: 200x100.png + :width: 100% + +Terminate replication controller with 100 pods on 200 nodes + +.. image:: 200x100_term.png + :width: 100% + +200 pods (~4 pod per core) on 200 nodes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Start replication controller with 200 pods on 200 nodes + +Note: Docker service is frozen on 27 nodes + +.. image:: 200x200.png + :width: 100% + +Terminate replication controller with 200 pods on 200 nodes + +.. image:: 200x200_term.png + :width: 100% + +400 pods (~8 pod per core) on 50 nodes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Start replication controller with 400 pods on 50 nodes + +.. image:: 50x400.png + :width: 100% + +Terminate replication controller with 400 pods on 50 nodes + +.. image:: 50x400_term.png + :width: 100% + .. references: .. _Kargo: https://github.com/kubespray/kargo diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/kubectl-mon/kubectl_mon_v2.py b/doc/source/test_results/container_cluster_systems/kubernetes/density/kubectl-mon/kubectl_mon_v2.py new file mode 100644 index 0000000..49f1efa --- /dev/null +++ b/doc/source/test_results/container_cluster_systems/kubernetes/density/kubectl-mon/kubectl_mon_v2.py @@ -0,0 +1,75 @@ +#!/bin/python + +import optparse +import re +import requests +from requests.adapters import HTTPAdapter +import sys +import subprocess +import time + +KUBECTL_CMD = 'kubectl --namespace minions get pods -l k8s-app=minion' +TIMEOUT = 1200 # in seconds + + +class Monitor(object): + def __init__(self, **options): + self.options = options + + def check_direction(self, n, l): + if self.direction == 'up': + if n > 0 and l == n + 1: + print 'Done.' + sys.exit(0) + if self.direction == 'down': + if l == 0: + print 'Done.' + sys.exit(0) + + def run(self): + self.direction = self.options.get('direction', 'up') + base_time = int(time.time() * 1000000000) + + s = requests.Session() + s.mount('http://master.minions:8888', HTTPAdapter(max_retries=10)) + + while True: + start = int(time.time() * 1000000000) + stdout = subprocess.Popen(KUBECTL_CMD, shell=True, + stdout=subprocess.PIPE).stdout.read() + + n = 0 + for line in stdout.split('\n')[1:]: + if line: + tokens = re.split('\s+', line) + name = tokens[0] + status = tokens[2] + + if status == 'Running': + n += 1 + + url = "http://master.minions:8888/monitor?minion_time=%s&" \ + "minion_name=%s&minion_status=%s&direction=%s" % ( + start, name, status, self.direction) + + s.get(url) + + length_lines = len(stdout.split('\n')[1:]) + self.check_direction(n, length_lines) + + if (start - base_time) / 1000000000 > TIMEOUT: + print 'Timeout.(%d)' % TIMEOUT + sys.exit(0) + + time.sleep(1) + + +def main(): + parser = optparse.OptionParser() + parser.add_option('-d', '--direction', dest='direction', default='up') + options, args = parser.parse_args() + + Monitor(**vars(options)).run() + +if __name__ == '__main__': + main()