From 09baadba2cc75a9f3b44c109b5fe49ca63d44b59 Mon Sep 17 00:00:00 2001 From: Yichen Wang Date: Wed, 4 Feb 2015 11:58:00 -0800 Subject: [PATCH] Initial release of VMTP to stackforge Change-Id: I30eb092d9a70dc6b3642a84887bb4604b1a3ea54 --- .coveragerc | 7 + .dockerignore | 8 + .gitignore | 59 +++ .mailmap | 3 + .testr.conf | 7 + CONTRIBUTING.rst | 16 + Dockerfile | 22 ++ HACKING.rst | 4 + LICENSE | 176 +++++++++ MANIFEST.in | 6 + README.md | 314 +++++++++++++++ README.rst | 29 ++ babel.cfg | 2 + cfg.default.yaml | 162 ++++++++ cfg.existing.yaml | 10 + cfg.nimbus.svl.yaml | 14 + cfg.nimbus.yaml | 9 + compute.py | 276 +++++++++++++ credentials.py | 110 ++++++ doc/source/conf.py | 75 ++++ doc/source/contributing.rst | 4 + doc/source/index.rst | 25 ++ doc/source/installation.rst | 12 + doc/source/readme.rst | 1 + doc/source/usage.rst | 7 + instance.py | 284 +++++++++++++ iperf_tool.py | 206 ++++++++++ monitor.py | 443 +++++++++++++++++++++ network.py | 262 ++++++++++++ nuttcp_tool.py | 194 +++++++++ openstack-common.conf | 6 + perf_instance.py | 105 +++++ perf_tool.py | 289 ++++++++++++++ pns_mongo.py | 142 +++++++ pnsdb_summary.py | 328 +++++++++++++++ pylintrc | 17 + requirements-dev.txt | 5 + requirements.txt | 20 + run_tests.sh | 2 + setup.cfg | 47 +++ setup.py | 22 ++ ssh/id_rsa | 27 ++ ssh/id_rsa.pub | 1 + sshutils.py | 475 ++++++++++++++++++++++ test-requirements.txt | 15 + tools/iperf | Bin 0 -> 68488 bytes tools/nuttcp-7.3.2 | Bin 0 -> 154592 bytes tox.ini | 42 ++ vmtp.py | 768 ++++++++++++++++++++++++++++++++++++ vmtp/__init__.py | 19 + vmtp/tests/__init__.py | 0 vmtp/tests/base.py | 23 ++ vmtp/tests/test_vmtp.py | 28 ++ 53 files changed, 5128 insertions(+) create mode 100644 .coveragerc create mode 100644 .dockerignore create mode 100644 .gitignore create mode 100644 .mailmap create mode 100644 .testr.conf create mode 100644 CONTRIBUTING.rst create mode 100644 Dockerfile create mode 100644 HACKING.rst create mode 100644 LICENSE create mode 100644 MANIFEST.in create mode 100644 README.md create mode 100644 README.rst create mode 100644 babel.cfg create mode 100644 cfg.default.yaml create mode 100644 cfg.existing.yaml create mode 100644 cfg.nimbus.svl.yaml create mode 100644 cfg.nimbus.yaml create mode 100644 compute.py create mode 100644 credentials.py create mode 100755 doc/source/conf.py create mode 100644 doc/source/contributing.rst create mode 100644 doc/source/index.rst create mode 100644 doc/source/installation.rst create mode 100644 doc/source/readme.rst create mode 100644 doc/source/usage.rst create mode 100644 instance.py create mode 100644 iperf_tool.py create mode 100755 monitor.py create mode 100755 network.py create mode 100644 nuttcp_tool.py create mode 100644 openstack-common.conf create mode 100644 perf_instance.py create mode 100644 perf_tool.py create mode 100755 pns_mongo.py create mode 100755 pnsdb_summary.py create mode 100644 pylintrc create mode 100644 requirements-dev.txt create mode 100644 requirements.txt create mode 100644 run_tests.sh create mode 100644 setup.cfg create mode 100755 setup.py create mode 100644 ssh/id_rsa create mode 100644 ssh/id_rsa.pub create mode 100644 sshutils.py create mode 100644 test-requirements.txt create mode 100755 tools/iperf create mode 100755 tools/nuttcp-7.3.2 create mode 100644 tox.ini create mode 100755 vmtp.py create mode 100644 vmtp/__init__.py create mode 100644 vmtp/tests/__init__.py create mode 100644 vmtp/tests/base.py create mode 100644 vmtp/tests/test_vmtp.py diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..444ba6d --- /dev/null +++ b/.coveragerc @@ -0,0 +1,7 @@ +[run] +branch = True +source = vmtp +omit = vmtp/tests/*,vmtp/openstack/* + +[report] +ignore-errors = True diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..1bd415a --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +ansible +installer +requirements-dev.txt +cloud_init* +.git +.gitignore +.gitreview +.pylintrc diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..58d97fd --- /dev/null +++ b/.gitignore @@ -0,0 +1,59 @@ +*.py[cod] + +# C extensions +*.so + +# Packages +*.egg +*.egg-info +dist +build +eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +lib +lib64 + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.coverage +.tox +nosetests.xml +.testrepository +.venv + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Complexity +output/*.html +output/*/index.html + +# Sphinx +doc/build + +# pbr generates these +AUTHORS +ChangeLog + +# Editors +*~ +.*.swp +.*sw? +*cscope* +.ropeproject/ + +# vmtp +*.local* +*.json diff --git a/.mailmap b/.mailmap new file mode 100644 index 0000000..516ae6f --- /dev/null +++ b/.mailmap @@ -0,0 +1,3 @@ +# Format is: +# +# diff --git a/.testr.conf b/.testr.conf new file mode 100644 index 0000000..6d83b3c --- /dev/null +++ b/.testr.conf @@ -0,0 +1,7 @@ +[DEFAULT] +test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \ + OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \ + OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \ + ${PYTHON:-python} -m subunit.run discover -t ./ . $LISTOPT $IDOPTION +test_id_option=--load-list $IDFILE +test_list_option=--list diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 0000000..53d09d7 --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,16 @@ +If you would like to contribute to the development of OpenStack, +you must follow the steps in this page: + + http://docs.openstack.org/infra/manual/developers.html + +Once those steps have been completed, changes to OpenStack +should be submitted for review via the Gerrit tool, following +the workflow documented at: + + http://docs.openstack.org/infra/manual/developers.html#development-workflow + +Pull requests submitted through GitHub will be ignored. + +Bugs should be filed on Launchpad, not GitHub: + + https://bugs.launchpad.net/vmtp diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c5e1591 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +# docker file for creating a container that has vmtp installed and ready to use +FROM ubuntu:14.04 +MAINTAINER openstack-systems-group + +# Install VMTP script and dependencies +RUN apt-get update && apt-get install -y \ + lib32z1-dev \ + libffi-dev \ + libssl-dev \ + libxml2-dev \ + libxslt1-dev \ + libyaml-dev \ + openssh-client \ + python \ + python-dev \ + python-lxml \ + python-pip + +COPY . /vmtp/ + +RUN pip install -r /vmtp/requirements.txt + diff --git a/HACKING.rst b/HACKING.rst new file mode 100644 index 0000000..4f2031d --- /dev/null +++ b/HACKING.rst @@ -0,0 +1,4 @@ +vmtp Style Commandments +=============================================== + +Read the OpenStack Style Commandments http://docs.openstack.org/developer/hacking/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..68c771a --- /dev/null +++ b/LICENSE @@ -0,0 +1,176 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..c978a52 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,6 @@ +include AUTHORS +include ChangeLog +exclude .gitignore +exclude .gitreview + +global-exclude *.pyc diff --git a/README.md b/README.md new file mode 100644 index 0000000..aec1e39 --- /dev/null +++ b/README.md @@ -0,0 +1,314 @@ + +# VMTP: An OpenStack TCP/UDP throughput measurement tool + +VMTP is a python application that will automatically perform ping connectivity, ping round trip time measuerment (latency) and TCP/UDP throughput measurement for the following flows on any OpenStack deployment: + +* VM to VM same network (private fixed IP) +* VM to VM different network same tenant (intra-tenant L3 fixed IP) +* VM to VM different network and tenant (floating IP inter-tenant L3) + +Optionally, when an external Linux host is available: + +* external host/VM download and upload throughput/latency (L3/floating IP) + +Optionally, when ssh login to any Linux host (native or virtual) is available: + +* host to host throughput (intra-node and inter-node) + +For VM-related flows, VMTP will automatically create the necessary OpenStack resources (router, networks, subnets, key pairs, security groups, test VMs), perform the throughput measurements then cleanup all related resources before exiting. +In the case involving pre-existing native or virtual hosts, VMTP will ssh to the targeted hosts to perform measurements. + +All TCP/UDP throughput measurements are done using the nuttcp tool by default. +The iperf tool can be used alternatively (--tp-tool iperf). + +Optionally, VMTP can extract automatically CPU usage from all native hosts in the cloud during the throughput tests, provided the Ganglia monitoring service (gmond) is installed and enabled on those hosts. + +Pre-requisite to run VMTP successfully: + +* For VM related performance measurements: + + * Access to the cloud Horizon Dashboard + * 1 working external network pre-configured on the cloud (VMTP will pick the first one found) + * at least 2 floating IP if an external router is configured or 3 floating IP if there is no external router configured + * 1 Linux image available in OpenStack (any distribution) + * a configuration file that is properly set for the cloud to test (see "Configuration File" section below) + +* for native/external host throughput, a public key must be installed on the target hosts (see ssh password-less access below) +* for pre-existing native host throughputs, firewalls must be configured to allow TCP/UDP ports 5001 and TCP port 5002 +* Docker if using the VMTP Docker image + +## VMTP results output + +VMTP will display the results to stdout with the following data: + +* session general information (date, auth_url, OpenStack encaps, VMTP version...) +* list of results per flow, for each flow: + + * flow name + * to and from IP addresses + * to and from availability zones (if VM) + * results: + * TCP + * throughput value + * number of retransmissions + * round trip time in ms + * CPU usage (if enabled), for each host in the openstack cluster: + * baseline (before test starts) + * 1 or more readings during test + * UDP + * for each packet size + * throughput value + * loss rate + * CPU usage (if enabled) + * ICMP + * average, min, max and stddev round trip time in ms + +Detailed results can also be stored in a file in JSON format using the --json command line argument. + +## How to run the VMTP tool + +### VMTP Docker image + +In its Docker image form, VMTP is located under the /vmtp directory in the container and can either take arguments from the host shell, or can be executed from inside the Docker image shell. + +To run VMTP directly from the host shell (may require "sudo" up front if not root) + +``` +docker run -i -t python /vmtp/vmtp.py + +``` + +To run VMTP from the Docker image shell: + +``` +docker run -i -t /bin/bash +cd /vmtp.py +python vmtp.py + +``` +(then type exit to exit and terminate the container instance) + +All the examples below assume running from inside the Docker image shell. + +### Print VMTP usage + +``` +usage: vmtp.py [-h] [-c ] [-r ] + [-m [:]] [-p ] [-t