Kubernetes dns perfomance test plan and report
Change-Id: Ib86a21171428f2c76e583bf94e5c579be723e05c
This commit is contained in:
Artem Yasakov
parent
324149914c
commit
856aa8e0fd
|
|
@ -0,0 +1,14 @@
|
|||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ name }}
|
||||
labels:
|
||||
name: {{ name }}
|
||||
spec:
|
||||
selector:
|
||||
name: {{ name }}
|
||||
ports:
|
||||
- port: 9999
|
||||
protocol: TCP
|
||||
targetPort: 9999
|
||||
type: NodePort
|
||||
|
|
@ -0,0 +1,133 @@
|
|||
import os
|
||||
import random
|
||||
import shlex
|
||||
import socket
|
||||
import string
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
|
||||
import jinja2
|
||||
|
||||
|
||||
LIST_SVC_SUCCESS = []
|
||||
LIST_SVC_FAILED = []
|
||||
RESULTS_FAILED = []
|
||||
|
||||
INTERVAL = 1 # in seconds
|
||||
SERVICES = 1000 # amount of services
|
||||
REQUESTS = 1000 # amount of requests
|
||||
|
||||
|
||||
def render(tpl_path, context):
|
||||
path, filename = os.path.split(tpl_path)
|
||||
return jinja2.Environment(
|
||||
loader=jinja2.FileSystemLoader(path or './')
|
||||
).get_template(filename).render(context)
|
||||
|
||||
|
||||
def id_generator(size=8, chars=string.ascii_lowercase + string.digits):
|
||||
return ''.join(random.choice(chars) for _ in range(size))
|
||||
|
||||
|
||||
def create_svc():
|
||||
service_name = "test-service-{}".format(id_generator())
|
||||
file_name = "{}.yaml".format(service_name)
|
||||
|
||||
# Create YAML file for new service
|
||||
template = render("template.yaml", {"name": service_name})
|
||||
f = open(file_name, "w")
|
||||
f.write(template)
|
||||
f.close()
|
||||
|
||||
cmd = "kubectl create -f {}".format(file_name)
|
||||
args = shlex.split(cmd)
|
||||
|
||||
proc = subprocess.Popen(args, stdout=subprocess.PIPE)
|
||||
(out, err) = proc.communicate()
|
||||
|
||||
# Delete YAML file
|
||||
os.remove(file_name)
|
||||
|
||||
if not err:
|
||||
print out
|
||||
LIST_SVC_SUCCESS.append(service_name)
|
||||
return True
|
||||
|
||||
print err
|
||||
LIST_SVC_FAILED.append(service_name)
|
||||
return False
|
||||
|
||||
|
||||
def delete_svc(service_name):
|
||||
cmd = "kubectl delete service {}".format(service_name)
|
||||
args = shlex.split(cmd)
|
||||
|
||||
proc = subprocess.Popen(args, stdout=subprocess.PIPE)
|
||||
(out, err) = proc.communicate()
|
||||
|
||||
if not err:
|
||||
print out
|
||||
return True
|
||||
|
||||
print err
|
||||
return False
|
||||
|
||||
|
||||
def request_to_host(service_name):
|
||||
host = "{}.default.svc.cluster.local".format(service_name)
|
||||
try:
|
||||
print socket.gethostbyname(host)
|
||||
except socket.gaierror:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def multi_requests():
|
||||
for i in range(0, REQUESTS):
|
||||
num_service = random.randint(0, len(LIST_SVC_SUCCESS) - 1)
|
||||
service_name = LIST_SVC_SUCCESS[num_service]
|
||||
if not request_to_host(service_name):
|
||||
RESULTS_FAILED.append(service_name)
|
||||
time.sleep(INTERVAL)
|
||||
|
||||
|
||||
def clean_up():
|
||||
for svc in LIST_SVC_SUCCESS:
|
||||
delete_svc(svc)
|
||||
|
||||
|
||||
def test_case(threads=50):
|
||||
query = []
|
||||
del RESULTS_FAILED[:]
|
||||
|
||||
for i in range(0, threads):
|
||||
query.append(threading.Thread(target=multi_requests, args=()))
|
||||
|
||||
for t in query:
|
||||
t.start()
|
||||
|
||||
for t in query:
|
||||
t.join()
|
||||
|
||||
return len(RESULTS_FAILED)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Create services
|
||||
for i in range(0, SERVICES):
|
||||
success = create_svc()
|
||||
|
||||
# Run test case
|
||||
for i in range(1, 21):
|
||||
print "{} rps".format(i * 50)
|
||||
result = test_case(i * 50)
|
||||
print "Wasn't resolve {} host(s)".format(result)
|
||||
|
||||
with open("result.txt", "a") as f:
|
||||
f.write("{} rps - wasn't resolve {} host(s)\n".format(
|
||||
i * 50, result))
|
||||
|
||||
# Delete created service
|
||||
clean_up()
|
||||
|
|
@ -0,0 +1,93 @@
|
|||
.. _Kubernetes_dns_performance_test_plan:
|
||||
|
||||
************************************
|
||||
Kubernetes DNS performance test plan
|
||||
************************************
|
||||
|
||||
:status: **ready**
|
||||
:version: 1.0
|
||||
|
||||
:Abstract:
|
||||
|
||||
This test plan covers scenarios for Kubernetes DNS performance testing.
|
||||
|
||||
Test Plan
|
||||
=========
|
||||
|
||||
Kubernetes DNS schedules a DNS Pod and Service on the cluster, and configures
|
||||
the kubelets to tell individual containers to use the DNS Service's IP to
|
||||
resolve DNS names.
|
||||
|
||||
"Normal" (not headless) Services are assigned a DNS A record for a name of
|
||||
the form ``my-svc.my-namespace.svc.cluster.local``. This resolves to the
|
||||
cluster IP of the Service.
|
||||
|
||||
Under DNS performance we mean the amount of work produced by a service,
|
||||
for DNS service this can be measured as number of requests for resolving host
|
||||
per second.
|
||||
|
||||
Test Environment
|
||||
----------------
|
||||
|
||||
Preparation
|
||||
^^^^^^^^^^^
|
||||
|
||||
The test plan is executed against Kubernetes deployed on bare-metal nodes.
|
||||
|
||||
Environment description
|
||||
^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
The environment description includes hardware specification of servers,
|
||||
network parameters, operation system and OpenStack deployment characteristics.
|
||||
|
||||
|
||||
Test Case #1: Performing DNS queries
|
||||
------------------------------------
|
||||
|
||||
Description
|
||||
^^^^^^^^^^^
|
||||
|
||||
In this test case we investigate how number of queries affects Kubernetes
|
||||
DNS performance.
|
||||
|
||||
Script :download:`code/test_kubedns.py` will create Kubernetes services based
|
||||
on file :download:`code/template.yaml`. After that, will make request to this
|
||||
services by host name. Results will show number of failed hosts.
|
||||
|
||||
Parameters
|
||||
^^^^^^^^^^
|
||||
|
||||
**Case group 1:**
|
||||
|
||||
.. table:
|
||||
|
||||
+----------------------+------------------------+
|
||||
| Parameter name | Value |
|
||||
+======================+========================+
|
||||
| number of replicas | 1, 2, 3 |
|
||||
+----------------------+------------------------+
|
||||
| requests per second | 50, 100, ..., 1000 |
|
||||
+----------------------+------------------------+
|
||||
| number of Services | 1000 |
|
||||
+----------------------+------------------------+
|
||||
| number of attempts | 1000 |
|
||||
+----------------------+------------------------+
|
||||
|
||||
List of performance metrics
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. table:: list of test metrics to be collected during this test
|
||||
|
||||
+-------------------------+-----------------------------------------------+
|
||||
| Parameter | Description |
|
||||
+=========================+===============================================+
|
||||
| FAILED_HOSTS | Number of hosts, which not be resolved by DNS |
|
||||
+-------------------------+-----------------------------------------------+
|
||||
| SUCCESS_RATE | Percentage of successful queries |
|
||||
+-------------------------+-----------------------------------------------+
|
||||
|
||||
Reports
|
||||
=======
|
||||
|
||||
Test plan execution reports:
|
||||
* :ref:`Kubernetes_dns_performance_test_report`
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 12 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 14 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 6.1 KiB |
|
|
@ -0,0 +1,267 @@
|
|||
.. _Kubernetes_dns_performance_test_report:
|
||||
|
||||
**************************************
|
||||
Kubernetes dns performance test report
|
||||
**************************************
|
||||
|
||||
:Abstract:
|
||||
|
||||
This document is the report for :ref:`Kubernetes_dns_performance_test_plan`
|
||||
|
||||
|
||||
Environment description
|
||||
=======================
|
||||
|
||||
This report is collected on the hardware described in
|
||||
:ref:`intel_mirantis_performance_lab_1`.
|
||||
|
||||
|
||||
Software
|
||||
~~~~~~~~
|
||||
|
||||
Kubernetes is installed using :ref:`Kargo` deployment tool on Ubuntu 16.04.1.
|
||||
|
||||
Node roles:
|
||||
- node1: minion+master+etcd
|
||||
- node2: minion+master+etcd
|
||||
- node3: minion+etcd
|
||||
- node4: minion
|
||||
- node5: minion
|
||||
- node6: minion
|
||||
|
||||
Software versions:
|
||||
- OS: Ubuntu 16.04.1 LTS (Xenial Xerus)
|
||||
- Kernel: 4.4.0-47-generic
|
||||
- Docker: 1.13.0
|
||||
- Kubernetes: v1.5.1+coreos.0 and v1.5.3+coreos.0
|
||||
|
||||
Reports
|
||||
=======
|
||||
|
||||
Test Case #1: Performing DNS queries
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Launched script where every next run uses parameters from previous one, but
|
||||
number of req/sec increases on 50.
|
||||
|
||||
|
||||
|
||||
Detailed Stats
|
||||
--------------
|
||||
|
||||
3 replicas of kubedns (v1.5.1)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. image:: chart.png
|
||||
|
||||
.. table:: Summary table performing DNS queries (v1.5.1)
|
||||
|
||||
+----+--------------------+---------------------+----------+--------------+--------------+
|
||||
| # | Number of services | Requests per second | Attempts | Failed hosts | Success rate |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 1 | 1000 | 50 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 2 | 1000 | 100 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 3 | 1000 | 150 | 1000 | 7 | 99.3% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 4 | 1000 | 200 | 1000 | 2 | 99.8% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 5 | 1000 | 250 | 1000 | 32 | 96.8% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 6 | 1000 | 300 | 1000 | 72 | 92.8% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 7 | 1000 | 350 | 1000 | 85 | 91.5% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 8 | 1000 | 400 | 1000 | 124 | 87.6% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 9 | 1000 | 450 | 1000 | 75 | 92.5% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 10 | 1000 | 500 | 1000 | 89 | 91.1% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 11 | 1000 | 550 | 1000 | 142 | 85.8% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 12 | 1000 | 600 | 1000 | 130 | 87% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 13 | 1000 | 650 | 1000 | 234 | 76.6% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 14 | 1000 | 700 | 1000 | 389 | 61.1% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 15 | 1000 | 750 | 1000 | 179 | 82.1% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 16 | 1000 | 800 | 1000 | 167 | 83.3% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 17 | 1000 | 850 | 1000 | 513 | 48.7% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 18 | 1000 | 900 | 1000 | 479 | 52.1% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 19 | 1000 | 950 | 1000 | 176 | 82.4% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 20 | 1000 | 1000 | 1000 | 622 | 37.8% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
|
||||
There was to increase performance for kubedns in new version Kubernetes(v1.5.3),
|
||||
therefore ran test again.
|
||||
|
||||
1 replicas of kubedns (v1.5.3)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. image:: chart2.png
|
||||
|
||||
.. table:: Summary table performing DNS queries (v1.5.3)
|
||||
|
||||
+----+--------------------+---------------------+----------+--------------+--------------+
|
||||
| # | Number of services | Requests per second | Attempts | Failed hosts | Success rate |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 1 | 1000 | 50 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 2 | 1000 | 100 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 3 | 1000 | 150 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 4 | 1000 | 200 | 1000 | 11 | 98.9% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 5 | 1000 | 250 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 6 | 1000 | 300 | 1000 | 26 | 97.4% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 7 | 1000 | 350 | 1000 | 31 | 96.9% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 8 | 1000 | 400 | 1000 | 13 | 98.7% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 9 | 1000 | 450 | 1000 | 92 | 90.8% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 10 | 1000 | 500 | 1000 | 24 | 97.6% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 11 | 1000 | 550 | 1000 | 98 | 90.2% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 12 | 1000 | 600 | 1000 | 49 | 95.1% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 13 | 1000 | 650 | 1000 | 226 | 77.4% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 14 | 1000 | 700 | 1000 | 163 | 83.7% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 15 | 1000 | 750 | 1000 | 5 | 99.5% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 16 | 1000 | 800 | 1000 | 54 | 94.6% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 17 | 1000 | 850 | 1000 | 178 | 82.2% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 18 | 1000 | 900 | 1000 | 113 | 88.7% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 19 | 1000 | 950 | 1000 | 150 | 85% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 20 | 1000 | 1000 | 1000 | 27 | 97.3% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
|
||||
2 replicas of kubedns (v1.5.3)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. image:: chart3.png
|
||||
|
||||
.. table:: Summary table performing DNS queries (v1.5.3)
|
||||
|
||||
+----+--------------------+---------------------+----------+--------------+--------------+
|
||||
| # | Number of services | Requests per second | Attempts | Failed hosts | Success rate |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 1 | 1000 | 50 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 2 | 1000 | 100 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 3 | 1000 | 150 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 4 | 1000 | 200 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 5 | 1000 | 250 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 6 | 1000 | 300 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 7 | 1000 | 350 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 8 | 1000 | 400 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 9 | 1000 | 450 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 10 | 1000 | 500 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 11 | 1000 | 550 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 12 | 1000 | 600 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 13 | 1000 | 650 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 14 | 1000 | 700 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 15 | 1000 | 750 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 16 | 1000 | 800 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 17 | 1000 | 850 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 18 | 1000 | 900 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 19 | 1000 | 950 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 20 | 1000 | 1000 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
|
||||
3 replicas of kubedns (v1.5.3)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. image:: chart3.png
|
||||
|
||||
.. table:: Summary table performing DNS queries (v1.5.3)
|
||||
|
||||
+----+--------------------+---------------------+----------+--------------+--------------+
|
||||
| # | Number of services | Requests per second | Attempts | Failed hosts | Success rate |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 1 | 1000 | 50 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 2 | 1000 | 100 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 3 | 1000 | 150 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 4 | 1000 | 200 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 5 | 1000 | 250 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 6 | 1000 | 300 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 7 | 1000 | 350 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 8 | 1000 | 400 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 9 | 1000 | 450 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 10 | 1000 | 500 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 11 | 1000 | 550 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 12 | 1000 | 600 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 13 | 1000 | 650 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 14 | 1000 | 700 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 15 | 1000 | 750 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 16 | 1000 | 800 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 17 | 1000 | 850 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 18 | 1000 | 900 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 19 | 1000 | 950 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 20 | 1000 | 1000 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 21 | 1000 | 1500 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 22 | 1000 | 2000 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 23 | 1000 | 2500 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 24 | 1000 | 3000 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
| 25 | 1000 | 3500 | 1000 | 0 | 100% |
|
||||
+-------------------------+---------------------+----------+--------------+--------------+
|
||||
Loading…
Reference in New Issue