diff --git a/api-ref/source/samples/db-mgmt-get-instance-details-response-json-http.txt b/api-ref/source/samples/db-mgmt-get-instance-details-response-json-http.txt index feb89a88ad..6580c3b803 100644 --- a/api-ref/source/samples/db-mgmt-get-instance-details-response-json-http.txt +++ b/api-ref/source/samples/db-mgmt-get-instance-details-response-json-http.txt @@ -1,5 +1,5 @@ HTTP/1.1 200 OK Content-Type: application/json -Content-Length: 1676 +Content-Length: 1709 Date: Mon, 18 Mar 2013 19:09:17 GMT diff --git a/api-ref/source/samples/db-mgmt-get-instance-details-response.json b/api-ref/source/samples/db-mgmt-get-instance-details-response.json index 203159d617..ef8b7efc9c 100644 --- a/api-ref/source/samples/db-mgmt-get-instance-details-response.json +++ b/api-ref/source/samples/db-mgmt-get-instance-details-response.json @@ -7,6 +7,7 @@ }, "deleted": false, "deleted_at": null, + "encrypted_rpc_messaging": true, "flavor": { "id": "3", "links": [ @@ -80,3 +81,4 @@ "volume_id": "VOL_44b277eb-39be-4921-be31-3d61b43651d7" } } + diff --git a/api-ref/source/samples/db-mgmt-instance-index-response-json-http.txt b/api-ref/source/samples/db-mgmt-instance-index-response-json-http.txt index 875f0f20a3..3994d592ce 100644 --- a/api-ref/source/samples/db-mgmt-instance-index-response-json-http.txt +++ b/api-ref/source/samples/db-mgmt-instance-index-response-json-http.txt @@ -1,5 +1,5 @@ HTTP/1.1 200 OK Content-Type: application/json -Content-Length: 1225 +Content-Length: 1258 Date: Mon, 18 Mar 2013 19:09:17 GMT diff --git a/api-ref/source/samples/db-mgmt-instance-index-response.json b/api-ref/source/samples/db-mgmt-instance-index-response.json index 5736bb17c0..6b26254a8e 100644 --- a/api-ref/source/samples/db-mgmt-instance-index-response.json +++ b/api-ref/source/samples/db-mgmt-instance-index-response.json @@ -8,6 +8,7 @@ }, "deleted": false, "deleted_at": null, + "encrypted_rpc_messaging": true, "flavor": { "id": "3", "links": [ @@ -58,3 +59,4 @@ } ] } + diff --git a/doc/source/dev/secure_oslo_messaging.rst b/doc/source/dev/secure_oslo_messaging.rst new file mode 100644 index 0000000000..beabd33938 --- /dev/null +++ b/doc/source/dev/secure_oslo_messaging.rst @@ -0,0 +1,655 @@ +.. _secure_rpc_messaging: + +====================== + Secure RPC messaging +====================== + +Background +---------- + +Trove uses oslo_messaging.rpc for communication amongst the various +control plane components and the guest agents. For secure operation of +the system, these RPC calls can be fully encrypted. A control plane +encryption key is used for communications between the API service and +the taskmanager, and system generated per-instance keys are used for +communication between the control plane and guest instances. + +This document provides some useful tips on how to use this mechanism. + +The default system behavior +--------------------------- + +By default, the system will attempt to encrypt all RPC +communication. This behavior is controlled by the following +configuration parameters: + +- enable_secure_rpc_messaging + + boolean that determines whether rpc messages will be secured by + encryption. The default value is True. + +- taskmanager_rpc_encr_key + + the key used for encrypting messages sent to the taskmanager. A + default value is provided for this and it is important that + deployers change this. + +- inst_rpc_key_encr_key + + the key used for encrypting the per-instance keys when they are + stored in the trove infrastructure database (catalog). A default is + provided for this and it is important that deployers change this. + + +Interoperability and Upgrade +---------------------------- + +Consider the system as shown below which runs a version of code prior +to the introduciton of this oslo_messaging.rpc security. Observe, for +example that the instances table in the system catalog does not +include the per-instance encrypted key column. + +mysql> describe instances; ++----------------------+--------------+------+-----+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++----------------------+--------------+------+-----+---------+-------+ +| id | varchar(36) | NO | PRI | NULL | | +| created | datetime | YES | | NULL | | +| updated | datetime | YES | | NULL | | +| name | varchar(255) | YES | | NULL | | +| hostname | varchar(255) | YES | | NULL | | +| compute_instance_id | varchar(36) | YES | | NULL | | +| task_id | int(11) | YES | | NULL | | +| task_description | varchar(255) | YES | | NULL | | +| task_start_time | datetime | YES | | NULL | | +| volume_id | varchar(36) | YES | | NULL | | +| flavor_id | varchar(255) | YES | | NULL | | +| volume_size | int(11) | YES | | NULL | | +| tenant_id | varchar(36) | YES | MUL | NULL | | +| server_status | varchar(64) | YES | | NULL | | +| deleted | tinyint(1) | YES | MUL | NULL | | +| deleted_at | datetime | YES | | NULL | | +| datastore_version_id | varchar(36) | NO | MUL | NULL | | +| configuration_id | varchar(36) | YES | MUL | NULL | | +| slave_of_id | varchar(36) | YES | MUL | NULL | | +| cluster_id | varchar(36) | YES | MUL | NULL | | +| shard_id | varchar(36) | YES | | NULL | | +| type | varchar(64) | YES | | NULL | | +| region_id | varchar(255) | YES | | NULL | | ++----------------------+--------------+------+-----+---------+-------+ +23 rows in set (0.00 sec) + +We launch an instance of MySQL using this version of the software. + +amrith@amrith-work:/opt/stack/trove/integration/scripts$ openstack network list ++--------------------------------------+-------------+--------------------------------------+ +| ID | Name | Subnets | ++--------------------------------------+-------------+--------------------------------------+ +[...] +| 4bab02e7-87bb-4cc0-8c07-2f282c777c85 | public | e620c4f5-749c-4212-b1d1-4a6e2c0a3f16 | +[...] ++--------------------------------------+-------------+--------------------------------------+ + +amrith@amrith-work:/opt/stack/trove/integration/scripts$ trove create m2 25 --size 3 --nic net-id=4bab02e7-87bb-4cc0-8c07-2f282c777c85 ++-------------------+--------------------------------------+ +| Property | Value | ++-------------------+--------------------------------------+ +| created | 2017-01-09T18:17:13 | +| datastore | mysql | +| datastore_version | 5.6 | +| flavor | 25 | +| id | bb0c9213-31f8-4427-8898-c644254b3642 | +| name | m2 | +| region | RegionOne | +| server_id | None | +| status | BUILD | +| updated | 2017-01-09T18:17:13 | +| volume | 3 | +| volume_id | None | ++-------------------+--------------------------------------+ + +amrith@amrith-work:/opt/stack/trove/integration/scripts$ nova list ++--------------------------------------+------+--------+------------+-------------+-------------------+ +| ID | Name | Status | Task State | Power State | Networks | ++--------------------------------------+------+--------+------------+-------------+-------------------+ +| a4769ce2-4e22-4134-b958-6db6c23cb221 | m2 | BUILD | spawning | NOSTATE | public=172.24.4.4 | ++--------------------------------------+------+--------+------------+-------------+-------------------+ + +And on that machine, the configuration file looks like this: + +amrith@m2:~$ cat /etc/trove/conf.d/guest_info.conf +[DEFAULT] +guest_id=bb0c9213-31f8-4427-8898-c644254b3642 +datastore_manager=mysql +tenant_id=56cca8484d3e48869126ada4f355c284 + +The instance goes online + +amrith@amrith-work:/opt/stack/trove/integration/scripts$ trove show m2 ++-------------------+--------------------------------------+ +| Property | Value | ++-------------------+--------------------------------------+ +| created | 2017-01-09T18:17:13 | +| datastore | mysql | +| datastore_version | 5.6 | +| flavor | 25 | +| id | bb0c9213-31f8-4427-8898-c644254b3642 | +| name | m2 | +| region | RegionOne | +| server_id | a4769ce2-4e22-4134-b958-6db6c23cb221 | +| status | ACTIVE | +| updated | 2017-01-09T18:17:17 | +| volume | 3 | +| volume_id | 16e57e3f-b462-4db2-968b-3c284aa2751c | +| volume_used | 0.11 | ++-------------------+--------------------------------------+ + +For testing later, we launch a few more instances. + +amrith@amrith-work:/opt/stack/trove/integration/scripts$ trove create m3 25 --size 3 --nic net-id=4bab02e7-87bb-4cc0-8c07-2f282c777c85 +amrith@amrith-work:/opt/stack/trove/integration/scripts$ trove create m4 25 --size 3 --nic net-id=4bab02e7-87bb-4cc0-8c07-2f282c777c85 + +amrith@amrith-work:/opt/stack/trove/integration/scripts$ trove list ++--------------------------------------+------+-----------+-------------------+--------+-----------+------+-----------+ +| ID | Name | Datastore | Datastore Version | Status | Flavor ID | Size | Region | ++--------------------------------------+------+-----------+-------------------+--------+-----------+------+-----------+ +| 6d55ab3a-267f-4b95-8ada-33fc98fd1767 | m4 | mysql | 5.6 | ACTIVE | 25 | 3 | RegionOne | +| 9ceebd62-e13d-43c5-953a-c0f24f08757e | m3 | mysql | 5.6 | ACTIVE | 25 | 3 | RegionOne | +| bb0c9213-31f8-4427-8898-c644254b3642 | m2 | mysql | 5.6 | ACTIVE | 25 | 3 | RegionOne | ++--------------------------------------+------+-----------+-------------------+--------+-----------+------+-----------+ + +In this condition, we take down the control plane and upgrade the +software running on it. This will result in a catalog upgrade. Since +this system is based on devstack, here's what that looks like. + +amrith@amrith-work:/opt/stack/trove$ git branch +* master + review/amrith/bp/secure-oslo-messaging-messages +amrith@amrith-work:/opt/stack/trove$ git checkout review/amrith/bp/secure-oslo-messaging-messages +Switched to branch 'review/amrith/bp/secure-oslo-messaging-messages' +Your branch is ahead of 'gerrit/master' by 1 commit. + (use "git push" to publish your local commits) +amrith@amrith-work:/opt/stack/trove$ find . -name '*.pyc' -delete +amrith@amrith-work:/opt/stack/trove$ + +amrith@amrith-work:/opt/stack/trove$ trove-manage db_sync +[...] +2017-01-09 13:24:25.251 DEBUG migrate.versioning.repository [-] Config: OrderedDict([('db_settings', OrderedDict([('__name__', 'db_settings'), ('repository_id', 'Trove Migrations'), ('version_table', 'migrate_version'), ('required_dbs', "['mysql','postgres','sqlite']")]))]) from (pid=96180) __init__ /usr/local/lib/python2.7/dist-packages/migrate/versioning/repository.py:83 +2017-01-09 13:24:25.260 INFO migrate.versioning.api [-] 40 -> 41... +2017-01-09 13:24:25.328 INFO migrate.versioning.api [-] done +2017-01-09 13:24:25.329 DEBUG migrate.versioning.util [-] Disposing SQLAlchemy engine Engine(mysql+pymysql://root:***@127.0.0.1/trove?charset=utf8) from (pid=96180) with_engine /usr/local/lib/python2.7/dist-packages/migrate/versioning/util/__init__.py:163 +[...] + +We observe that the new table in the system has the encrypted_key column + +mysql> describe instances; ++----------------------+--------------+------+-----+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++----------------------+--------------+------+-----+---------+-------+ +| id | varchar(36) | NO | PRI | NULL | | +| created | datetime | YES | | NULL | | +| updated | datetime | YES | | NULL | | +| name | varchar(255) | YES | | NULL | | +| hostname | varchar(255) | YES | | NULL | | +| compute_instance_id | varchar(36) | YES | | NULL | | +| task_id | int(11) | YES | | NULL | | +| task_description | varchar(255) | YES | | NULL | | +| task_start_time | datetime | YES | | NULL | | +| volume_id | varchar(36) | YES | | NULL | | +| flavor_id | varchar(255) | YES | | NULL | | +| volume_size | int(11) | YES | | NULL | | +| tenant_id | varchar(36) | YES | MUL | NULL | | +| server_status | varchar(64) | YES | | NULL | | +| deleted | tinyint(1) | YES | MUL | NULL | | +| deleted_at | datetime | YES | | NULL | | +| datastore_version_id | varchar(36) | NO | MUL | NULL | | +| configuration_id | varchar(36) | YES | MUL | NULL | | +| slave_of_id | varchar(36) | YES | MUL | NULL | | +| cluster_id | varchar(36) | YES | MUL | NULL | | +| shard_id | varchar(36) | YES | | NULL | | +| type | varchar(64) | YES | | NULL | | +| region_id | varchar(255) | YES | | NULL | | +| encrypted_key | varchar(255) | YES | | NULL | | ++----------------------+--------------+------+-----+---------+-------+ + + +mysql> select id, encrypted_key from instances; ++--------------------------------------+---------------+ +| id | encrypted_key | ++--------------------------------------+---------------+ +| 13a787f2-b699-4867-a727-b3f4d8040a12 | NULL | ++--------------------------------------+---------------+ +1 row in set (0.00 sec) + +amrith@amrith-work:/opt/stack/trove$ sudo python setup.py install -f +[...] + +We can now relaunch the control plane software but before we do that, +we inspect the configuration parameters and disable secure RPC +messaging by adding this line into the configuration files. + +amrith@amrith-work:/etc/trove$ grep enable_secure_rpc_messaging *.conf +trove-conductor.conf:enable_secure_rpc_messaging = False +trove.conf:enable_secure_rpc_messaging = False +trove-taskmanager.conf:enable_secure_rpc_messaging = False + +The first thing we observe is that heartbeat messages from the +existing instance are still properly handled by the conductor and the +instance remains active. + +2017-01-09 13:26:57.742 DEBUG oslo_messaging._drivers.amqpdriver [-] received message with unique_id: eafe22c08bae485e9346ce0fbdaa4d6c from (pid=96551) __call__ /usr/local/lib/python2.7/dist-packages/oslo_messaging/_drivers/amqpdriver.py:196 +2017-01-09 13:26:57.744 DEBUG trove.conductor.manager [-] Instance ID: bb0c9213-31f8-4427-8898-c644254b3642, Payload: {u'service_status': u'running'} from (pid=96551) heartbeat /opt/stack/trove/trove/conductor/manager.py:88 +2017-01-09 13:26:57.748 DEBUG trove.conductor.manager [-] Instance bb0c9213-31f8-4427-8898-c644254b3642 sent heartbeat at 1483986416.52 from (pid=96551) _message_too_old /opt/stack/trove/trove/conductor/manager.py:54 +2017-01-09 13:26:57.750 DEBUG trove.conductor.manager [-] [Instance bb0c9213-31f8-4427-8898-c644254b3642] Rec'd message is younger than last seen. Updating. from (pid=96551) _message_too_old /opt/stack/trove/trove/conductor/manager.py:76 +2017-01-09 13:27:01.197 DEBUG oslo_messaging._drivers.amqpdriver [-] received message with unique_id: df62b76523004338876bc7b08f8b7711 from (pid=96552) __call__ /usr/local/lib/python2.7/dist-packages/oslo_messaging/_drivers/amqpdriver.py:196 +2017-01-09 13:27:01.200 DEBUG trove.conductor.manager [-] Instance ID: 9ceebd62-e13d-43c5-953a-c0f24f08757e, Payload: {u'service_status': u'running'} from (pid=96552) heartbeat /opt/stack/trove/trove/conductor/manager.py:88 +2017-01-09 13:27:01.219 DEBUG oslo_db.sqlalchemy.engines [-] Parent process 96542 forked (96552) with an open database connection, which is being discarded and recreated. from (pid=96552) checkout /usr/local/lib/python2.7/dist-packages/oslo_db/sqlalchemy/engines.py:362 +2017-01-09 13:27:01.225 DEBUG trove.conductor.manager [-] Instance 9ceebd62-e13d-43c5-953a-c0f24f08757e sent heartbeat at 1483986419.99 from (pid=96552) _message_too_old /opt/stack/trove/trove/conductor/manager.py:54 +2017-01-09 13:27:01.231 DEBUG trove.conductor.manager [-] [Instance 9ceebd62-e13d-43c5-953a-c0f24f08757e] Rec'd message is younger than last seen. Updating. from (pid=96552) _message_too_old /opt/stack/trove/trove/conductor/manager.py:76 + +amrith@amrith-work:/etc/trove$ trove list ++--------------------------------------+------+-----------+-------------------+--------+-----------+------+-----------+ +| ID | Name | Datastore | Datastore Version | Status | Flavor ID | Size | Region | ++--------------------------------------+------+-----------+-------------------+--------+-----------+------+-----------+ +| 6d55ab3a-267f-4b95-8ada-33fc98fd1767 | m4 | mysql | 5.6 | ACTIVE | 25 | 3 | RegionOne | +| 9ceebd62-e13d-43c5-953a-c0f24f08757e | m3 | mysql | 5.6 | ACTIVE | 25 | 3 | RegionOne | +| bb0c9213-31f8-4427-8898-c644254b3642 | m2 | mysql | 5.6 | ACTIVE | 25 | 3 | RegionOne | ++--------------------------------------+------+-----------+-------------------+--------+-----------+------+-----------+ + +amrith@amrith-work:/etc/trove$ trove show m2 ++-------------------+--------------------------------------+ +| Property | Value | ++-------------------+--------------------------------------+ +| created | 2017-01-09T18:17:13 | +| datastore | mysql | +| datastore_version | 5.6 | +| flavor | 25 | +| id | bb0c9213-31f8-4427-8898-c644254b3642 | +| name | m2 | +| region | RegionOne | +| server_id | a4769ce2-4e22-4134-b958-6db6c23cb221 | +| status | ACTIVE | +| updated | 2017-01-09T18:17:17 | +| volume | 3 | +| volume_id | 16e57e3f-b462-4db2-968b-3c284aa2751c | +| volume_used | 0.11 | ++-------------------+--------------------------------------+ + +We now launch a new instance, recall that secure_rpc_messaging is disabled. + +amrith@amrith-work:/etc/trove$ trove create m10 25 --size 3 --nic net-id=4bab02e7-87bb-4cc0-8c07-2f282c777c85 ++-------------------+--------------------------------------+ +| Property | Value | ++-------------------+--------------------------------------+ +| created | 2017-01-09T18:28:56 | +| datastore | mysql | +| datastore_version | 5.6 | +| flavor | 25 | +| id | 514ef051-0bf7-48a5-adcf-071d4a6625fb | +| name | m10 | +| region | RegionOne | +| server_id | None | +| status | BUILD | +| updated | 2017-01-09T18:28:56 | +| volume | 3 | +| volume_id | None | ++-------------------+--------------------------------------+ + +Observe that the task manager does not create a password for the instance. + +2017-01-09 13:29:00.111 INFO trove.instance.models [-] Resetting task status to NONE on instance 514ef051-0bf7-48a5-adcf-071d4a6625fb. +2017-01-09 13:29:00.115 DEBUG trove.db.models [-] Saving DBInstance: {u'region_id': u'RegionOne', u'cluster_id': None, u'shard_id': None, u'deleted_at': None, u'id': u'514ef051-0bf7-48a5-adcf-071d4a6625fb', u'datastore_version_id': u'4a881cb5-9e48-4cb2-a209-4283ed44eb01', 'errors': {}, u'hostname': None, u'server_status': None, u'task_description': u'No tasks for the instance.', u'volume_size': 3, u'type': None, u'updated': datetime.datetime(2017, 1, 9, 18, 29, 0, 114971), '_sa_instance_state': , u'encrypted_key': None, u'deleted': 0, u'configuration_id': None, u'volume_id': u'cee2e17b-80fa-48e5-a488-da8b7809373a', u'slave_of_id': None, u'task_start_time': None, u'name': u'm10', u'task_id': 1, u'created': datetime.datetime(2017, 1, 9, 18, 28, 56), u'tenant_id': u'56cca8484d3e48869126ada4f355c284', u'compute_instance_id': u'2452263e-3d33-48ec-8f24-2851fe74db28', u'flavor_id': u'25'} from (pid=96635) save /opt/stack/trove/trove/db/models.py:64 + + +the configuration file for this instance is: + +amrith@m10:~$ cat /etc/trove/conf.d/guest_info.conf +[DEFAULT] +guest_id=514ef051-0bf7-48a5-adcf-071d4a6625fb +datastore_manager=mysql +tenant_id=56cca8484d3e48869126ada4f355c284 + +We can now shutdown the control plane again and enable the secure RPC +capability. Observe that we've just commented out the lines (below). + +trove-conductor.conf:# enable_secure_rpc_messaging = False +trove.conf:# enable_secure_rpc_messaging = False +trove-taskmanager.conf:# enable_secure_rpc_messaging = False + +And create another database instance + +amrith@amrith-work:/etc/trove$ trove create m20 25 --size 3 --nic net-id=4bab02e7-87bb-4cc0-8c07-2f282c777c85 ++-------------------+--------------------------------------+ +| Property | Value | ++-------------------+--------------------------------------+ +| created | 2017-01-09T18:31:48 | +| datastore | mysql | +| datastore_version | 5.6 | +| flavor | 25 | +| id | 792fa220-2a40-4831-85af-cfb0ded8033c | +| name | m20 | +| region | RegionOne | +| server_id | None | +| status | BUILD | +| updated | 2017-01-09T18:31:48 | +| volume | 3 | +| volume_id | None | ++-------------------+--------------------------------------+ + +Observe that a unique per-instance encryption key was created for this instance. + +2017-01-09 13:31:52.474 DEBUG trove.db.models [-] Saving DBInstance: {u'region_id': u'RegionOne', u'cluster_id': None, u'shard_id': None, u'deleted_at': None, u'id': u'792fa220-2a40-4831-85af-cfb0ded8033c', u'datastore_version_id': u'4a881cb5-9e48-4cb2-a209-4283ed44eb01', 'errors': {}, u'hostname': None, u'server_status': None, u'task_description': u'No tasks for the instance.', u'volume_size': 3, u'type': None, u'updated': datetime.datetime(2017, 1, 9, 18, 31, 52, 473552), '_sa_instance_state': , u'encrypted_key': u'fVpHrkUIjVsXe7Fj7Lm4u2xnJUsWX2rMC9GL0AppILJINBZxLvkowY8FOa+asKS+8pWb4iNyukQQ4AQoLEUHUQ==', u'deleted': 0, u'configuration_id': None, u'volume_id': u'4cd563dc-fe08-477b-828f-120facf4351b', u'slave_of_id': None, u'task_start_time': None, u'name': u'm20', u'task_id': 1, u'created': datetime.datetime(2017, 1, 9, 18, 31, 49), u'tenant_id': u'56cca8484d3e48869126ada4f355c284', u'compute_instance_id': u'1e62a192-83d3-43fd-b32e-b5ee2fa4e24b', u'flavor_id': u'25'} from (pid=97562) save /opt/stack/trove/trove/db/models.py:64 + +And the configuration file on that instance includes an encryption key. + +amrith@m20:~$ cat /etc/trove/conf.d/guest_info.conf +[DEFAULT] +guest_id=792fa220-2a40-4831-85af-cfb0ded8033c +datastore_manager=mysql +tenant_id=56cca8484d3e48869126ada4f355c284 +instance_rpc_encr_key=eRz43LwE6eaxIbBlA2pNukzPjSdcQkVi + +amrith@amrith-work:/etc/trove$ trove list ++--------------------------------------+------+-----------+-------------------+--------+-----------+------+-----------+ +| ID | Name | Datastore | Datastore Version | Status | Flavor ID | Size | Region | ++--------------------------------------+------+-----------+-------------------+--------+-----------+------+-----------+ +| 514ef051-0bf7-48a5-adcf-071d4a6625fb | m10 | mysql | 5.6 | ACTIVE | 25 | 3 | RegionOne | +| 6d55ab3a-267f-4b95-8ada-33fc98fd1767 | m4 | mysql | 5.6 | ACTIVE | 25 | 3 | RegionOne | +| 792fa220-2a40-4831-85af-cfb0ded8033c | m20 | mysql | 5.6 | ACTIVE | 25 | 3 | RegionOne | +| 9ceebd62-e13d-43c5-953a-c0f24f08757e | m3 | mysql | 5.6 | ACTIVE | 25 | 3 | RegionOne | +| bb0c9213-31f8-4427-8898-c644254b3642 | m2 | mysql | 5.6 | ACTIVE | 25 | 3 | RegionOne | ++--------------------------------------+------+-----------+-------------------+--------+-----------+------+-----------+ + +At this point communication between API service and Task Manager, and +between the control plane and instance m20 is encrypted but +communication between control plane and all other instances is not +encrypted. + +In this condition we can attempt some operations on the various +instances. First with the legacy instances created on software that +predated the secure RPC mechanism. + +amrith@amrith-work:/etc/trove$ trove database-list m2 ++------+ +| Name | ++------+ ++------+ +amrith@amrith-work:/etc/trove$ trove database-create m2 foo2 +amrith@amrith-work:/etc/trove$ trove database-list m2 ++------+ +| Name | ++------+ +| foo2 | ++------+ + +And at the same time with the instance m10 which is created with the +current software but without RPC encryption. + +amrith@amrith-work:/etc/trove$ trove database-list m10 ++------+ +| Name | ++------+ ++------+ +amrith@amrith-work:/etc/trove$ trove database-create m10 foo10 +amrith@amrith-work:/etc/trove$ trove database-list m10 ++-------+ +| Name | ++-------+ +| foo10 | ++-------+ +amrith@amrith-work:/etc/trove$ + +And finally with an instance that uses encrypted RPC communications. + +amrith@amrith-work:/etc/trove$ trove database-list m20 ++------+ +| Name | ++------+ ++------+ +amrith@amrith-work:/etc/trove$ trove database-create m20 foo20 +amrith@amrith-work:/etc/trove$ trove database-list m20 ++-------+ +| Name | ++-------+ +| foo20 | ++-------+ + +Finally, we can upgrade an instance that has no encryption to have rpc +encryption. + +amrith@amrith-work:/etc/trove$ trove datastore-list ++--------------------------------------+------------------+ +| ID | Name | ++--------------------------------------+------------------+ +| 8e052edb-5f14-4aec-9149-0a80a30cf5e4 | mysql | ++--------------------------------------+------------------+ +amrith@amrith-work:/etc/trove$ trove datastore-version-list mysql ++--------------------------------------+------------------+ +| ID | Name | ++--------------------------------------+------------------+ +| 4a881cb5-9e48-4cb2-a209-4283ed44eb01 | 5.6 | ++--------------------------------------+------------------+ + +Let's look at instance m2. + +mysql> select id, name, encrypted_key from instances where id = 'bb0c9213-31f8-4427-8898-c644254b3642'; ++--------------------------------------+------+---------------+ +| id | name | encrypted_key | ++--------------------------------------+------+---------------+ +| bb0c9213-31f8-4427-8898-c644254b3642 | m2 | NULL | ++--------------------------------------+------+---------------+ +1 row in set (0.00 sec) + +amrith@amrith-work:/etc/trove$ trove upgrade m2 4a881cb5-9e48-4cb2-a209-4283ed44eb01 + +amrith@amrith-work:/etc/trove$ trove list ++--------------------------------------+------+-----------+-------------------+---------+-----------+------+-----------+ +| ID | Name | Datastore | Datastore Version | Status | Flavor ID | Size | Region | ++--------------------------------------+------+-----------+-------------------+---------+-----------+------+-----------+ +| 514ef051-0bf7-48a5-adcf-071d4a6625fb | m10 | mysql | 5.6 | ACTIVE | 25 | 3 | RegionOne | +| 6d55ab3a-267f-4b95-8ada-33fc98fd1767 | m4 | mysql | 5.6 | ACTIVE | 25 | 3 | RegionOne | +| 792fa220-2a40-4831-85af-cfb0ded8033c | m20 | mysql | 5.6 | ACTIVE | 25 | 3 | RegionOne | +| 9ceebd62-e13d-43c5-953a-c0f24f08757e | m3 | mysql | 5.6 | ACTIVE | 25 | 3 | RegionOne | +| bb0c9213-31f8-4427-8898-c644254b3642 | m2 | mysql | 5.6 | UPGRADE | 25 | 3 | RegionOne | ++--------------------------------------+------+-----------+-------------------+---------+-----------+------+-----------+ + +amrith@amrith-work:/etc/trove$ nova list ++--------------------------------------+------+---------+------------+-------------+--------------------+ +| ID | Name | Status | Task State | Power State | Networks | ++--------------------------------------+------+---------+------------+-------------+--------------------+ +[...] +| a4769ce2-4e22-4134-b958-6db6c23cb221 | m2 | REBUILD | rebuilding | Running | public=172.24.4.4 | +[...] ++--------------------------------------+------+---------+------------+-------------+--------------------+ + + +2017-01-09 13:47:24.337 DEBUG trove.db.models [-] Saving DBInstance: {u'region_id': u'RegionOne', u'cluster_id': None, u'shard_id': None, u'deleted_at': None, u'id': u'bb0c9213-31f8-4427-8898-c644254b3642', u'datastore_version_id': u'4a881cb5-9e48-4cb2-a209-4283ed44eb01', 'errors': {}, u'hostname': None, u'server_status': None, u'task_description': u'Upgrading the instance.', u'volume_size': 3, u'type': None, u'updated': datetime.datetime(2017, 1, 9, 18, 47, 24, 337400), '_sa_instance_state': , u'encrypted_key': u'gMrlHkEVxKgEFMTabzZr2TLJ6r5+wgfJfhohs7K/BzutWxs1wXfBswyV5Bgw4qeD212msmgSdOUCFov5otgzyg==', u'deleted': 0, u'configuration_id': None, u'volume_id': u'16e57e3f-b462-4db2-968b-3c284aa2751c', u'slave_of_id': None, u'task_start_time': None, u'name': u'm2', u'task_id': 89, u'created': datetime.datetime(2017, 1, 9, 18, 17, 13), u'tenant_id': u'56cca8484d3e48869126ada4f355c284', u'compute_instance_id': u'a4769ce2-4e22-4134-b958-6db6c23cb221', u'flavor_id': u'25'} from (pid=97562) save /opt/stack/trove/trove/db/models.py:64 +2017-01-09 13:47:24.347 DEBUG trove.taskmanager.models [-] Generated unique RPC encryption key for instance = bb0c9213-31f8-4427-8898-c644254b3642, key = gMrlHkEVxKgEFMTabzZr2TLJ6r5+wgfJfhohs7K/BzutWxs1wXfBswyV5Bgw4qeD212msmgSdOUCFov5otgzyg== from (pid=97562) upgrade /opt/stack/trove/trove/taskmanager/models.py:1440 +2017-01-09 13:47:24.350 DEBUG trove.taskmanager.models [-] Rebuilding instance m2(bb0c9213-31f8-4427-8898-c644254b3642) with image ea05cba7-2f70-4745-abea-136d7bcc16c7. from (pid=97562) upgrade /opt/stack/trove/trove/taskmanager/models.py:1445 + +The instance now has an encryption key in its configuration + +amrith@m2:~$ cat /etc/trove/conf.d/guest_info.conf +[DEFAULT] +guest_id=bb0c9213-31f8-4427-8898-c644254b3642 +datastore_manager=mysql +tenant_id=56cca8484d3e48869126ada4f355c284 +instance_rpc_encr_key=pN2hHEl171ngyD0mPvyV1xKJF2im01Gv + +amrith@amrith-work:/etc/trove$ trove list ++--------------------------------------+------+-----------+-------------------+--------+-----------+------+-----------+ +| ID | Name | Datastore | Datastore Version | Status | Flavor ID | Size | Region | ++--------------------------------------+------+-----------+-------------------+--------+-----------+------+-----------+ +[...] +| bb0c9213-31f8-4427-8898-c644254b3642 | m2 | mysql | 5.6 | ACTIVE | 25 | 3 | RegionOne | +[...] ++--------------------------------------+------+-----------+-------------------+--------+-----------+------+-----------+ + +amrith@amrith-work:/etc/trove$ trove show m2 ++-------------------+--------------------------------------+ +| Property | Value | ++-------------------+--------------------------------------+ +| created | 2017-01-09T18:17:13 | +| datastore | mysql | +| datastore_version | 5.6 | +| flavor | 25 | +| id | bb0c9213-31f8-4427-8898-c644254b3642 | +| name | m2 | +| region | RegionOne | +| server_id | a4769ce2-4e22-4134-b958-6db6c23cb221 | +| status | ACTIVE | +| updated | 2017-01-09T18:50:07 | +| volume | 3 | +| volume_id | 16e57e3f-b462-4db2-968b-3c284aa2751c | +| volume_used | 0.13 | ++-------------------+--------------------------------------+ + +amrith@amrith-work:/etc/trove$ trove database-list m2 ++------+ +| Name | ++------+ +| foo2 | ++------+ + +We can similarly upgrade m4. + +2017-01-09 13:51:43.078 DEBUG trove.instance.models [-] Instance 6d55ab3a-267f-4b95-8ada-33fc98fd1767 service status is running. from (pid=97562) load_instance /opt/stack/trove/trove/instance/models.py:534 +2017-01-09 13:51:43.083 DEBUG trove.taskmanager.models [-] Upgrading instance m4(6d55ab3a-267f-4b95-8ada-33fc98fd1767) to new datastore version 5.6(4a881cb5-9e48-4cb2-a209-4283ed44eb01) from (pid=97562) upgrade /opt/stack/trove/trove/taskmanager/models.py:1410 +2017-01-09 13:51:43.087 DEBUG trove.guestagent.api [-] Sending the call to prepare the guest for upgrade. from (pid=97562) pre_upgrade /opt/stack/trove/trove/guestagent/api.py:351 +2017-01-09 13:51:43.087 DEBUG trove.guestagent.api [-] Calling pre_upgrade with timeout 600 from (pid=97562) _call /opt/stack/trove/trove/guestagent/api.py:86 +2017-01-09 13:51:43.088 DEBUG oslo_messaging._drivers.amqpdriver [-] CALL msg_id: 41dbb7fff3dc4f8fa69d8b5f219809e0 exchange 'trove' topic 'guestagent.6d55ab3a-267f-4b95-8ada-33fc98fd1767' from (pid=97562) _send /usr/local/lib/python2.7/dist-packages/oslo_messaging/_drivers/amqpdriver.py:442 +2017-01-09 13:51:45.452 DEBUG oslo_messaging._drivers.amqpdriver [-] received reply msg_id: 41dbb7fff3dc4f8fa69d8b5f219809e0 from (pid=97562) __call__ /usr/local/lib/python2.7/dist-packages/oslo_messaging/_drivers/amqpdriver.py:299 +2017-01-09 13:51:45.452 DEBUG trove.guestagent.api [-] Result is {u'mount_point': u'/var/lib/mysql', u'save_etc_dir': u'/var/lib/mysql/etc', u'home_save': u'/var/lib/mysql/trove_user', u'save_dir': u'/var/lib/mysql/etc_mysql'}. from (pid=97562) _call /opt/stack/trove/trove/guestagent/api.py:91 +2017-01-09 13:51:45.544 DEBUG trove.db.models [-] Saving DBInstance: {u'region_id': u'RegionOne', u'cluster_id': None, u'shard_id': None, u'deleted_at': None, u'id': u'6d55ab3a-267f-4b95-8ada-33fc98fd1767', u'datastore_version_id': u'4a881cb5-9e48-4cb2-a209-4283ed44eb01', 'errors': {}, u'hostname': None, u'server_status': None, u'task_description': u'Upgrading the instance.', u'volume_size': 3, u'type': None, u'updated': datetime.datetime(2017, 1, 9, 18, 51, 45, 544496), '_sa_instance_state': , u'encrypted_key': u'0gBkJl5Aqb4kFIPeJDMTNIymEUuUUB8NBksecTiYyQl+Ibrfi7ME8Bi58q2n61AxbG2coOqp97ETjHRyN7mYTg==', u'deleted': 0, u'configuration_id': None, u'volume_id': u'b7dc17b5-d0a8-47bb-aef4-ef9432c269e9', u'slave_of_id': None, u'task_start_time': None, u'name': u'm4', u'task_id': 89, u'created': datetime.datetime(2017, 1, 9, 18, 20, 58), u'tenant_id': u'56cca8484d3e48869126ada4f355c284', u'compute_instance_id': u'f43bba63-3be6-4993-b2d0-4ddfb7818d27', u'flavor_id': u'25'} from (pid=97562) save /opt/stack/trove/trove/db/models.py:64 +2017-01-09 13:51:45.557 DEBUG trove.taskmanager.models [-] Generated unique RPC encryption key for instance = 6d55ab3a-267f-4b95-8ada-33fc98fd1767, key = 0gBkJl5Aqb4kFIPeJDMTNIymEUuUUB8NBksecTiYyQl+Ibrfi7ME8Bi58q2n61AxbG2coOqp97ETjHRyN7mYTg== from (pid=97562) upgrade /opt/stack/trove/trove/taskmanager/models.py:1440 +2017-01-09 13:51:45.560 DEBUG trove.taskmanager.models [-] Rebuilding instance m4(6d55ab3a-267f-4b95-8ada-33fc98fd1767) with image ea05cba7-2f70-4745-abea-136d7bcc16c7. from (pid=97562) upgrade /opt/stack/trove/trove/taskmanager/models.py:1445 + +amrith@amrith-work:/etc/trove$ nova list ++--------------------------------------+------+---------+------------+-------------+--------------------+ +| ID | Name | Status | Task State | Power State | Networks | ++--------------------------------------+------+---------+------------+-------------+--------------------+ +[...] +| f43bba63-3be6-4993-b2d0-4ddfb7818d27 | m4 | REBUILD | rebuilding | Running | public=172.24.4.11 | +[...] ++--------------------------------------+------+---------+------------+-------------+--------------------+ + +2017-01-09 13:53:26.581 DEBUG trove.guestagent.api [-] Recover the guest after upgrading the guest's image. from (pid=97562) post_upgrade /opt/stack/trove/trove/guestagent/api.py:359 +2017-01-09 13:53:26.581 DEBUG trove.guestagent.api [-] Recycling the client ... from (pid=97562) post_upgrade /opt/stack/trove/trove/guestagent/api.py:361 +2017-01-09 13:53:26.581 DEBUG trove.guestagent.api [-] Calling post_upgrade with timeout 600 from (pid=97562) _call /opt/stack/trove/trove/guestagent/api.py:86 +2017-01-09 13:53:26.583 DEBUG oslo_messaging._drivers.amqpdriver [-] CALL msg_id: 2e9ccc88715b4b98848a017e19b2938d exchange 'trove' topic 'guestagent.6d55ab3a-267f-4b95-8ada-33fc98fd1767' from (pid=97562) _send /usr/local/lib/python2.7/dist-packages/oslo_messaging/_drivers/amqpdriver.py:442 + +mysql> select id, name, encrypted_key from instances where name in ('m2', 'm4', 'm10', 'm20'); ++--------------------------------------+------+------------------------------------------------------------------------------------------+ +| id | name | encrypted_key | ++--------------------------------------+------+------------------------------------------------------------------------------------------+ +| 514ef051-0bf7-48a5-adcf-071d4a6625fb | m10 | NULL | +| 6d55ab3a-267f-4b95-8ada-33fc98fd1767 | m4 | 0gBkJl5Aqb4kFIPeJDMTNIymEUuUUB8NBksecTiYyQl+Ibrfi7ME8Bi58q2n61AxbG2coOqp97ETjHRyN7mYTg== | +| 792fa220-2a40-4831-85af-cfb0ded8033c | m20 | fVpHrkUIjVsXe7Fj7Lm4u2xnJUsWX2rMC9GL0AppILJINBZxLvkowY8FOa+asKS+8pWb4iNyukQQ4AQoLEUHUQ== | +| bb0c9213-31f8-4427-8898-c644254b3642 | m2 | gMrlHkEVxKgEFMTabzZr2TLJ6r5+wgfJfhohs7K/BzutWxs1wXfBswyV5Bgw4qeD212msmgSdOUCFov5otgzyg== | ++--------------------------------------+------+------------------------------------------------------------------------------------------+ + +amrith@amrith-work:/etc/trove$ trove list ++--------------------------------------+------+-----------+-------------------+--------+-----------+------+-----------+ +| ID | Name | Datastore | Datastore Version | Status | Flavor ID | Size | Region | ++--------------------------------------+------+-----------+-------------------+--------+-----------+------+-----------+ +| 514ef051-0bf7-48a5-adcf-071d4a6625fb | m10 | mysql | 5.6 | ACTIVE | 25 | 3 | RegionOne | +| 6d55ab3a-267f-4b95-8ada-33fc98fd1767 | m4 | mysql | 5.6 | ACTIVE | 25 | 3 | RegionOne | +| 792fa220-2a40-4831-85af-cfb0ded8033c | m20 | mysql | 5.6 | ACTIVE | 25 | 3 | RegionOne | +| bb0c9213-31f8-4427-8898-c644254b3642 | m2 | mysql | 5.6 | ACTIVE | 25 | 3 | RegionOne | ++--------------------------------------+------+-----------+-------------------+--------+-----------+------+-----------+ + +Inspecting which instances are using secure RPC communications +-------------------------------------------------------------- + +An additional field is returned in the trove show command output to +indicate whether any given instance is using secure RPC communication +or not. + +NOTE: This field is only returned if the user is an 'admin'. Non admin +users do not see the field. + +amrith@amrith-work:/opt/stack/trove$ trove show m20 ++-------------------------+--------------------------------------+ +| Property | Value | ++-------------------------+--------------------------------------+ +| created | 2017-01-09T18:31:49 | +| datastore | mysql | +| datastore_version | 5.6 | +| encrypted_rpc_messaging | True | +| flavor | 25 | +| id | 792fa220-2a40-4831-85af-cfb0ded8033c | +| name | m20 | +| region | RegionOne | +| server_id | 1e62a192-83d3-43fd-b32e-b5ee2fa4e24b | +| status | ACTIVE | +| updated | 2017-01-09T18:31:52 | +| volume | 3 | +| volume_id | 4cd563dc-fe08-477b-828f-120facf4351b | +| volume_used | 0.11 | ++-------------------------+--------------------------------------+ +amrith@amrith-work:/opt/stack/trove$ trove show m10 ++-------------------------+--------------------------------------+ +| Property | Value | ++-------------------------+--------------------------------------+ +| created | 2017-01-09T18:28:56 | +| datastore | mysql | +| datastore_version | 5.6 | +| encrypted_rpc_messaging | False | +| flavor | 25 | +| id | 514ef051-0bf7-48a5-adcf-071d4a6625fb | +| name | m10 | +| region | RegionOne | +| server_id | 2452263e-3d33-48ec-8f24-2851fe74db28 | +| status | ACTIVE | +| updated | 2017-01-09T18:29:00 | +| volume | 3 | +| volume_id | cee2e17b-80fa-48e5-a488-da8b7809373a | +| volume_used | 0.11 | ++-------------------------+--------------------------------------+ +amrith@amrith-work:/opt/stack/trove$ trove show m2 ++-------------------------+--------------------------------------+ +| Property | Value | ++-------------------------+--------------------------------------+ +| created | 2017-01-09T18:17:13 | +| datastore | mysql | +| datastore_version | 5.6 | +| encrypted_rpc_messaging | True | +| flavor | 25 | +| id | bb0c9213-31f8-4427-8898-c644254b3642 | +| name | m2 | +| region | RegionOne | +| server_id | a4769ce2-4e22-4134-b958-6db6c23cb221 | +| status | ACTIVE | +| updated | 2017-01-09T18:50:07 | +| volume | 3 | +| volume_id | 16e57e3f-b462-4db2-968b-3c284aa2751c | +| volume_used | 0.13 | ++-------------------------+--------------------------------------+ +amrith@amrith-work:/opt/stack/trove$ trove show m4 ++-------------------------+--------------------------------------+ +| Property | Value | ++-------------------------+--------------------------------------+ +| created | 2017-01-09T18:20:58 | +| datastore | mysql | +| datastore_version | 5.6 | +| encrypted_rpc_messaging | True | +| flavor | 25 | +| id | 6d55ab3a-267f-4b95-8ada-33fc98fd1767 | +| name | m4 | +| region | RegionOne | +| server_id | f43bba63-3be6-4993-b2d0-4ddfb7818d27 | +| status | ACTIVE | +| updated | 2017-01-09T18:54:30 | +| volume | 3 | +| volume_id | b7dc17b5-d0a8-47bb-aef4-ef9432c269e9 | +| volume_used | 0.13 | ++-------------------------+--------------------------------------+ +amrith@amrith-work:/opt/stack/trove$ + +In the API response, note that the additional key +"encrypted_rpc_messaging" has been added (as below). + +NOTE: This field is only returned if the user is an 'admin'. Non admin +users do not see the field. + +RESP BODY: {"instance": {"status": "ACTIVE", "updated": "2017-01-09T18:29:00", "name": "m10", "links": [{"href": "https://192.168.126.130:8779/v1.0/56cca8484d3e48869126ada4f355c284/instances/514ef051-0bf7-48a5-adcf-071d4a6625fb", "rel": "self"}, {"href": "https://192.168.126.130:8779/instances/514ef051-0bf7-48a5-adcf-071d4a6625fb", "rel": "bookmark"}], "created": "2017-01-09T18:28:56", "region": "RegionOne", "server_id": "2452263e-3d33-48ec-8f24-2851fe74db28", "id": "514ef051-0bf7-48a5-adcf-071d4a6625fb", "volume": {"used": 0.11, "size": 3}, "volume_id": "cee2e17b-80fa-48e5-a488-da8b7809373a", "flavor": {"id": "25", "links": [{"href": "https://192.168.126.130:8779/v1.0/56cca8484d3e48869126ada4f355c284/flavors/25", "rel": "self"}, {"href": "https://192.168.126.130:8779/flavors/25", "rel": "bookmark"}]}, "datastore": {"version": "5.6", "type": "mysql"}, "encrypted_rpc_messaging": false}} diff --git a/doc/source/index.rst b/doc/source/index.rst index f803a3764e..ed1511d1f0 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -51,6 +51,7 @@ functionality, the following resources are provided. dev/guest_cloud_init.rst dev/notifier.rst dev/trove_api_extensions.rst + dev/secure_oslo_messaging.rst * Source Code Repositories diff --git a/run_tests.py b/run_tests.py index 5f4c98eea7..eb00e03236 100644 --- a/run_tests.py +++ b/run_tests.py @@ -76,7 +76,8 @@ def initialize_trove(config_file): rpc.init(CONF) taskman_service = rpc_service.RpcService( - None, topic=topic, rpc_api_version=rpc_version.RPC_API_VERSION, + CONF.taskmanager_rpc_encr_key, topic=topic, + rpc_api_version=rpc_version.RPC_API_VERSION, manager='trove.taskmanager.manager.Manager') taskman_service.start() diff --git a/tools/trove-pylint.config b/tools/trove-pylint.config index b67f5fa126..fad1504995 100644 --- a/tools/trove-pylint.config +++ b/tools/trove-pylint.config @@ -729,6 +729,18 @@ "Instance of 'Table' has no 'create_column' member", "upgrade" ], + [ + "trove/db/sqlalchemy/migrate_repo/versions/041_instance_keys.py", + "E1101", + "Instance of 'Table' has no 'create_column' member", + "upgrade" + ], + [ + "trove/db/sqlalchemy/migrate_repo/versions/041_instance_keys.py", + "no-member", + "Instance of 'Table' has no 'create_column' member", + "upgrade" + ], [ "trove/db/sqlalchemy/migration.py", "E0611", @@ -1107,12 +1119,24 @@ "Class 'InstanceStatus' has no 'LOGGING' member", "SimpleInstance.status" ], + [ + "trove/instance/models.py", + "E1101", + "Instance of 'DBInstance' has no 'encrypted_key' member", + "DBInstance.key" + ], [ "trove/instance/models.py", "no-member", "Class 'InstanceStatus' has no 'LOGGING' member", "SimpleInstance.status" ], + [ + "trove/instance/models.py", + "no-member", + "Instance of 'DBInstance' has no 'encrypted_key' member", + "DBInstance.key" + ], [ "trove/instance/service.py", "E1101", diff --git a/trove/cmd/conductor.py b/trove/cmd/conductor.py index daff5df4e5..793ad6b606 100644 --- a/trove/cmd/conductor.py +++ b/trove/cmd/conductor.py @@ -22,6 +22,7 @@ from trove.conductor import api as conductor_api @with_initialize def main(conf): from trove.common import notification + from trove.common.rpc import conductor_host_serializer as sz from trove.common.rpc import service as rpc_service from trove.instance import models as inst_models @@ -29,8 +30,9 @@ def main(conf): inst_models.persist_instance_fault) topic = conf.conductor_queue server = rpc_service.RpcService( - manager=conf.conductor_manager, topic=topic, - rpc_api_version=conductor_api.API.API_LATEST_VERSION) + key=None, manager=conf.conductor_manager, topic=topic, + rpc_api_version=conductor_api.API.API_LATEST_VERSION, + secure_serializer=sz.ConductorHostSerializer) workers = conf.trove_conductor_workers or processutils.get_worker_count() launcher = openstack_service.launch(conf, server, workers=workers) launcher.wait() diff --git a/trove/cmd/fakemode.py b/trove/cmd/fakemode.py index 66e5b3cdae..e66431fdd4 100644 --- a/trove/cmd/fakemode.py +++ b/trove/cmd/fakemode.py @@ -54,7 +54,7 @@ def start_fake_taskmanager(conf): from trove.common.rpc import service as rpc_service from trove.common.rpc import version as rpc_version taskman_service = rpc_service.RpcService( - topic=topic, rpc_api_version=rpc_version.RPC_API_VERSION, + key='', topic=topic, rpc_api_version=rpc_version.RPC_API_VERSION, manager='trove.taskmanager.manager.Manager') taskman_service.start() diff --git a/trove/cmd/guest.py b/trove/cmd/guest.py index ccb33563c5..19692d1483 100644 --- a/trove/cmd/guest.py +++ b/trove/cmd/guest.py @@ -30,13 +30,15 @@ from trove.guestagent import api as guest_api CONF = cfg.CONF # The guest_id opt definition must match the one in common/cfg.py CONF.register_opts([openstack_cfg.StrOpt('guest_id', default=None, - help="ID of the Guest Instance.")]) + help="ID of the Guest Instance."), + openstack_cfg.StrOpt('instance_rpc_encr_key', + help=('Key (OpenSSL aes_cbc) for ' + 'instance RPC encryption.'))]) def main(): cfg.parse_args(sys.argv) logging.setup(CONF, None) - debug_utils.setup() from trove.guestagent import dbaas @@ -51,6 +53,9 @@ def main(): "was not injected into the guest or not read by guestagent")) raise RuntimeError(msg) + # BUG(1650518): Cleanup in the Pike release + # make it fatal if CONF.instance_rpc_encr_key is None + # rpc module must be loaded after decision about thread monkeypatching # because if thread module is not monkeypatched we can't use eventlet # executor from oslo_messaging library. @@ -59,6 +64,7 @@ def main(): from trove.common.rpc import service as rpc_service server = rpc_service.RpcService( + key=CONF.instance_rpc_encr_key, topic="guestagent.%s" % CONF.guest_id, manager=manager, host=CONF.guest_id, rpc_api_version=guest_api.API.API_LATEST_VERSION) diff --git a/trove/cmd/taskmanager.py b/trove/cmd/taskmanager.py index aaef017c66..549e14b292 100644 --- a/trove/cmd/taskmanager.py +++ b/trove/cmd/taskmanager.py @@ -29,8 +29,14 @@ def startup(conf, topic): notification.DBaaSAPINotification.register_notify_callback( inst_models.persist_instance_fault) + + if conf.enable_secure_rpc_messaging: + key = conf.taskmanager_rpc_encr_key + else: + key = None + server = rpc_service.RpcService( - manager=conf.taskmanager_manager, topic=topic, + key=key, manager=conf.taskmanager_manager, topic=topic, rpc_api_version=task_api.API.API_LATEST_VERSION) launcher = openstack_service.launch(conf, server) launcher.wait() diff --git a/trove/common/cfg.py b/trove/common/cfg.py index b4b0c655e2..2005165071 100644 --- a/trove/common/cfg.py +++ b/trove/common/cfg.py @@ -444,6 +444,16 @@ common_opts = [ help='Maximum size of a chunk saved in guest log container.'), cfg.IntOpt('guest_log_expiry', default=2592000, help='Expiry (in seconds) of objects in guest log container.'), + cfg.BoolOpt('enable_secure_rpc_messaging', default=True, + help='Should RPC messaging traffic be secured by encryption.'), + cfg.StrOpt('taskmanager_rpc_encr_key', + default='bzH6y0SGmjuoY0FNSTptrhgieGXNDX6PIhvz', + help='Key (OpenSSL aes_cbc) for taskmanager RPC encryption.'), + cfg.StrOpt('inst_rpc_key_encr_key', + default='emYjgHFqfXNB1NGehAFIUeoyw4V4XwWHEaKP', + help='Key (OpenSSL aes_cbc) to encrypt instance keys in DB.'), + cfg.StrOpt('instance_rpc_encr_key', + help='Key (OpenSSL aes_cbc) for instance RPC encryption.'), ] diff --git a/trove/common/context.py b/trove/common/context.py index 254993d8a7..73d8a8581f 100644 --- a/trove/common/context.py +++ b/trove/common/context.py @@ -39,6 +39,7 @@ class TroveContext(context.RequestContext): self.marker = kwargs.pop('marker', None) self.service_catalog = kwargs.pop('service_catalog', None) self.user_identity = kwargs.pop('user_identity', None) + self.instance_id = kwargs.pop('instance_id', None) # TODO(esp): not sure we need this self.timeout = kwargs.pop('timeout', None) diff --git a/trove/common/crypto_utils.py b/trove/common/crypto_utils.py index bd8e3fb088..9e3d5613bc 100644 --- a/trove/common/crypto_utils.py +++ b/trove/common/crypto_utils.py @@ -20,7 +20,9 @@ from Crypto.Cipher import AES from Crypto import Random import hashlib from oslo_utils import encodeutils +import random import six +import string from trove.common import stream_codecs @@ -68,3 +70,9 @@ def decrypt_data(data, key, iv_bit_count=IV_BIT_COUNT): aes = AES.new(md5_key, AES.MODE_CBC, bytes(iv)) decrypted = aes.decrypt(bytes(data[iv_bit_count:])) return unpad_after_decryption(decrypted) + + +def generate_random_key(length=32, chars=None): + chars = chars if chars else (string.ascii_uppercase + + string.ascii_lowercase + string.digits) + return ''.join(random.choice(chars) for _ in range(length)) diff --git a/trove/common/rpc/conductor_guest_serializer.py b/trove/common/rpc/conductor_guest_serializer.py new file mode 100644 index 0000000000..e3b8afa356 --- /dev/null +++ b/trove/common/rpc/conductor_guest_serializer.py @@ -0,0 +1,60 @@ +# Copyright 2016 Tesora, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg +from oslo_serialization import jsonutils + +from trove.common import crypto_utils as crypto +from trove.common.i18n import _ +from trove.common.rpc import serializer + +CONF = cfg.CONF + + +# BUG(1650518): Cleanup in the Pike release +class ConductorGuestSerializer(serializer.TroveSerializer): + def __init__(self, base, key): + self._key = key + super(ConductorGuestSerializer, self).__init__(base) + + def _serialize_entity(self, ctxt, entity): + if self._key is None: + return entity + + value = crypto.encode_data( + crypto.encrypt_data( + jsonutils.dumps(entity), self._key)) + + return jsonutils.dumps({'entity': value, 'csz-instance-id': + CONF.guest_id}) + + def _deserialize_entity(self, ctxt, entity): + msg = (_("_deserialize_entity not implemented in " + "ConductorGuestSerializer.")) + raise Exception(msg) + + def _serialize_context(self, ctxt): + if self._key is None: + return ctxt + + cstr = jsonutils.dumps(ctxt) + + return {'context': + crypto.encode_data( + crypto.encrypt_data(cstr, self._key)), + 'csz-instance-id': CONF.guest_id} + + def _deserialize_context(self, ctxt): + msg = (_("_deserialize_context not implemented in " + "ConductorGuestSerializer.")) + raise Exception(msg) diff --git a/trove/common/rpc/conductor_host_serializer.py b/trove/common/rpc/conductor_host_serializer.py new file mode 100644 index 0000000000..0e17efd214 --- /dev/null +++ b/trove/common/rpc/conductor_host_serializer.py @@ -0,0 +1,83 @@ +# Copyright 2016 Tesora, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg +from oslo_serialization import jsonutils + +from trove.common import crypto_utils as cu +from trove.common.rpc import serializer +from trove.instance.models import get_instance_encryption_key + +CONF = cfg.CONF + + +# BUG(1650518): Cleanup in the Pike release +class ConductorHostSerializer(serializer.TroveSerializer): + def __init__(self, base, *_): + super(ConductorHostSerializer, self).__init__(base) + + def _serialize_entity(self, ctxt, entity): + try: + if ctxt.instance_id is None: + return entity + except (ValueError, TypeError): + return entity + + instance_key = get_instance_encryption_key(ctxt.instance_id) + + estr = jsonutils.dumps(entity) + return cu.encode_data(cu.encrypt_data(estr, instance_key)) + + def _deserialize_entity(self, ctxt, entity): + try: + entity = jsonutils.loads(entity) + instance_id = entity['csz-instance-id'] + except (ValueError, TypeError): + return entity + + instance_key = get_instance_encryption_key(instance_id) + + estr = cu.decrypt_data(cu.decode_data(entity['entity']), + instance_key) + entity = jsonutils.loads(estr) + + return entity + + def _serialize_context(self, ctxt): + try: + if ctxt.instance_id is None: + return ctxt + except (ValueError, TypeError): + return ctxt + + instance_key = get_instance_encryption_key(ctxt.instance_id) + + cstr = jsonutils.dumps(ctxt) + return {'context': cu.encode_data(cu.encrypt_data(cstr, + instance_key))} + + def _deserialize_context(self, ctxt): + try: + instance_id = ctxt.get('csz-instance-id', None) + + if instance_id is not None: + instance_key = get_instance_encryption_key(instance_id) + + cstr = cu.decrypt_data(cu.decode_data(ctxt['context']), + instance_key) + ctxt = jsonutils.loads(cstr) + except (ValueError, TypeError): + return ctxt + + ctxt['instance_id'] = instance_id + return ctxt diff --git a/trove/common/rpc/secure_serializer.py b/trove/common/rpc/secure_serializer.py new file mode 100644 index 0000000000..3430b939cc --- /dev/null +++ b/trove/common/rpc/secure_serializer.py @@ -0,0 +1,59 @@ +# Copyright 2016 Tesora, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_serialization import jsonutils + +from trove.common import crypto_utils as cu +from trove.common.rpc import serializer + + +# BUG(1650518): Cleanup in the Pike release +class SecureSerializer(serializer.TroveSerializer): + def __init__(self, base, key): + self._key = key + super(SecureSerializer, self).__init__(base) + + def _serialize_entity(self, ctxt, entity): + if self._key is None: + return entity + + estr = jsonutils.dumps(entity) + return cu.encode_data(cu.encrypt_data(estr, self._key)) + + def _deserialize_entity(self, ctxt, entity): + try: + if self._key is not None: + estr = cu.decrypt_data(cu.decode_data(entity), self._key) + entity = jsonutils.loads(estr) + except (ValueError, TypeError): + return entity + + return entity + + def _serialize_context(self, ctxt): + if self._key is None: + return ctxt + + cstr = jsonutils.dumps(ctxt) + return {'context': cu.encode_data(cu.encrypt_data(cstr, self._key))} + + def _deserialize_context(self, ctxt): + try: + if self._key is not None: + cstr = cu.decrypt_data(cu.decode_data(ctxt['context']), + self._key) + ctxt = jsonutils.loads(cstr) + except (ValueError, TypeError): + return ctxt + + return ctxt diff --git a/trove/common/rpc/serializer.py b/trove/common/rpc/serializer.py new file mode 100644 index 0000000000..0073f29339 --- /dev/null +++ b/trove/common/rpc/serializer.py @@ -0,0 +1,86 @@ +# Copyright 2016 Tesora, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import oslo_messaging as messaging +from osprofiler import profiler + +from trove.common.context import TroveContext + + +class TroveSerializer(messaging.Serializer): + """The Trove serializer class that handles class inheritence and base + serializers. + """ + + def __init__(self, base): + self._base = base + + def _serialize_entity(self, context, entity): + return entity + + def serialize_entity(self, context, entity): + if self._base: + entity = self._base.serialize_entity(context, entity) + + return self._serialize_entity(context, entity) + + def _deserialize_entity(self, context, entity): + return entity + + def deserialize_entity(self, context, entity): + entity = self._deserialize_entity(context, entity) + + if self._base: + entity = self._base.deserialize_entity(context, entity) + + return entity + + def _serialize_context(self, context): + return context + + def serialize_context(self, context): + if self._base: + context = self._base.serialize_context(context) + + return self._serialize_context(context) + + def _deserialize_context(self, context): + return context + + def deserialize_context(self, context): + context = self._deserialize_context(context) + + if self._base: + context = self._base.deserialize_context(context) + + return context + + +class TroveRequestContextSerializer(TroveSerializer): + def _serialize_context(self, context): + _context = context.to_dict() + prof = profiler.get() + if prof: + trace_info = { + "hmac_key": prof.hmac_key, + "base_id": prof.get_base_id(), + "parent_id": prof.get_id() + } + _context.update({"trace_info": trace_info}) + return _context + + def _deserialize_context(self, context): + trace_info = context.pop("trace_info", None) + if trace_info: + profiler.init(**trace_info) + return TroveContext.from_dict(context) diff --git a/trove/common/rpc/service.py b/trove/common/rpc/service.py index f5ff2af844..ed3924c022 100644 --- a/trove/common/rpc/service.py +++ b/trove/common/rpc/service.py @@ -29,6 +29,7 @@ from osprofiler import profiler from trove.common import cfg from trove.common.i18n import _ from trove.common import profile +from trove.common.rpc import secure_serializer as ssz from trove import rpc @@ -38,9 +39,10 @@ LOG = logging.getLogger(__name__) class RpcService(service.Service): - def __init__(self, host=None, binary=None, topic=None, manager=None, - rpc_api_version=None): + def __init__(self, key, host=None, binary=None, topic=None, manager=None, + rpc_api_version=None, secure_serializer=ssz.SecureSerializer): super(RpcService, self).__init__() + self.key = key self.host = host or CONF.host self.binary = binary or os.path.basename(inspect.stack()[-1][1]) self.topic = topic or self.binary.rpartition('trove-')[2] @@ -48,6 +50,7 @@ class RpcService(service.Service): self.manager_impl = profiler.trace_cls("rpc")(_manager) self.rpc_api_version = rpc_api_version or \ self.manager_impl.RPC_API_VERSION + self.secure_serializer = secure_serializer profile.setup_profiler(self.binary, self.host) def start(self): @@ -60,7 +63,9 @@ class RpcService(service.Service): self.manager_impl.target = target endpoints = [self.manager_impl] - self.rpcserver = rpc.get_server(target, endpoints) + self.rpcserver = rpc.get_server( + target, endpoints, key=self.key, + secure_serializer=self.secure_serializer) self.rpcserver.start() # TODO(hub-cap): Currently the context is none... do we _need_ it here? diff --git a/trove/conductor/api.py b/trove/conductor/api.py index 757416b225..be73b2b751 100644 --- a/trove/conductor/api.py +++ b/trove/conductor/api.py @@ -16,6 +16,7 @@ from oslo_log import log as logging import oslo_messaging as messaging from trove.common import cfg +from trove.common.rpc import conductor_guest_serializer as sz from trove.common.serializable_notification import SerializableNotification from trove import rpc @@ -62,9 +63,10 @@ class API(object): self.client = self.get_client(target, version_cap) def get_client(self, target, version_cap, serializer=None): - return rpc.get_client(target, + return rpc.get_client(target, key=CONF.instance_rpc_encr_key, version_cap=version_cap, - serializer=serializer) + serializer=serializer, + secure_serializer=sz.ConductorGuestSerializer) def heartbeat(self, instance_id, payload, sent=None): LOG.debug("Making async call to cast heartbeat for instance: %s" diff --git a/trove/db/models.py b/trove/db/models.py index 6b8e0475e6..90dc4800d3 100644 --- a/trove/db/models.py +++ b/trove/db/models.py @@ -13,6 +13,7 @@ # under the License. from oslo_log import log as logging +from oslo_utils import strutils from trove.common import exception from trove.common.i18n import _ @@ -59,13 +60,15 @@ class DatabaseModelBase(models.ModelBase): raise exception.InvalidModelError(errors=self.errors) self['updated'] = utils.utcnow() LOG.debug("Saving %(name)s: %(dict)s" % - {'name': self.__class__.__name__, 'dict': self.__dict__}) + {'name': self.__class__.__name__, + 'dict': strutils.mask_dict_password(self.__dict__)}) return self.db_api.save(self) def delete(self): self['updated'] = utils.utcnow() LOG.debug("Deleting %(name)s: %(dict)s" % - {'name': self.__class__.__name__, 'dict': self.__dict__}) + {'name': self.__class__.__name__, + 'dict': strutils.mask_dict_password(self.__dict__)}) if self.preserve_on_delete: self['deleted_at'] = utils.utcnow() diff --git a/trove/db/sqlalchemy/migrate_repo/versions/041_instance_keys.py b/trove/db/sqlalchemy/migrate_repo/versions/041_instance_keys.py new file mode 100644 index 0000000000..7477cfaf71 --- /dev/null +++ b/trove/db/sqlalchemy/migrate_repo/versions/041_instance_keys.py @@ -0,0 +1,30 @@ +# Copyright 2016 Tesora, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +from sqlalchemy.schema import Column +from sqlalchemy.schema import MetaData + +from trove.db.sqlalchemy.migrate_repo.schema import String +from trove.db.sqlalchemy.migrate_repo.schema import Table + + +meta = MetaData() + + +def upgrade(migrate_engine): + meta.bind = migrate_engine + instances = Table('instances', meta, autoload=True) + instances.create_column(Column('encrypted_key', String(255))) diff --git a/trove/guestagent/api.py b/trove/guestagent/api.py index 180388a07d..85be70fa82 100644 --- a/trove/guestagent/api.py +++ b/trove/guestagent/api.py @@ -69,13 +69,16 @@ class API(object): version_cap = self.VERSION_ALIASES.get( CONF.upgrade_levels.guestagent, CONF.upgrade_levels.guestagent) - target = messaging.Target(topic=self._get_routing_key(), - version=version_cap) + self.target = messaging.Target(topic=self._get_routing_key(), + version=version_cap) - self.client = self.get_client(target, version_cap) + self.client = self.get_client(self.target, version_cap) def get_client(self, target, version_cap, serializer=None): - return rpc.get_client(target, + from trove.instance.models import get_instance_encryption_key + + instance_key = get_instance_encryption_key(self.id) + return rpc.get_client(target, key=instance_key, version_cap=version_cap, serializer=serializer) @@ -328,12 +331,15 @@ class API(object): method do nothing in case a queue is already created by the guest """ + from trove.instance.models import DBInstance server = None target = messaging.Target(topic=self._get_routing_key(), server=self.id, version=self.API_BASE_VERSION) try: - server = rpc.get_server(target, []) + instance = DBInstance.get_by(id=self.id) + instance_key = instance.key if instance else None + server = rpc.get_server(target, [], key=instance_key) server.start() finally: if server is not None: @@ -352,6 +358,10 @@ class API(object): """Recover the guest after upgrading the guest's image.""" LOG.debug("Recover the guest after upgrading the guest's image.") version = self.API_BASE_VERSION + LOG.debug("Recycling the client ...") + version_cap = self.VERSION_ALIASES.get( + CONF.upgrade_levels.guestagent, CONF.upgrade_levels.guestagent) + self.client = self.get_client(self.target, version_cap) self._call("post_upgrade", AGENT_HIGH_TIMEOUT, version=version, upgrade_info=upgrade_info) diff --git a/trove/instance/models.py b/trove/instance/models.py index d5cc1519b9..11656712ab 100644 --- a/trove/instance/models.py +++ b/trove/instance/models.py @@ -26,6 +26,7 @@ from oslo_log import log as logging from trove.backup.models import Backup from trove.common import cfg +from trove.common import crypto_utils as cu from trove.common import exception from trove.common.glance_remote import create_glance_client from trove.common.i18n import _, _LE, _LI, _LW @@ -433,6 +434,10 @@ class SimpleInstance(object): def region_name(self): return self.db_info.region_id + @property + def encrypted_rpc_messaging(self): + return True if self.db_info.encrypted_key is not None else False + class DetailInstance(SimpleInstance): """A detailed view of an Instance. @@ -749,6 +754,14 @@ class BaseInstance(SimpleInstance): "tenant_id=%s\n" % (self.id, datastore_manager, self.tenant_id))} + instance_key = get_instance_encryption_key(self.id) + if instance_key: + files = {guest_info_file: ( + "%s" + "instance_rpc_encr_key=%s\n" % ( + files.get(guest_info_file), + instance_key))} + if os.path.isfile(CONF.get('guest_config')): with open(CONF.get('guest_config'), "r") as f: files[os.path.join(injected_config_location, @@ -1502,7 +1515,8 @@ class DBInstance(dbmodels.DatabaseModelBase): 'task_id', 'task_description', 'task_start_time', 'volume_id', 'deleted', 'tenant_id', 'datastore_version_id', 'configuration_id', 'slave_of_id', - 'cluster_id', 'shard_id', 'type', 'region_id'] + 'cluster_id', 'shard_id', 'type', 'region_id', + 'encrypted_key'] def __init__(self, task_status, **kwargs): """ @@ -1515,9 +1529,27 @@ class DBInstance(dbmodels.DatabaseModelBase): kwargs["task_id"] = task_status.code kwargs["task_description"] = task_status.db_text kwargs["deleted"] = False + + if CONF.enable_secure_rpc_messaging: + key = cu.generate_random_key() + kwargs["encrypted_key"] = cu.encode_data(cu.encrypt_data( + key, CONF.inst_rpc_key_encr_key)) + LOG.debug("Generated unique RPC encryption key for " + "instance. key = %s" % key) + else: + kwargs["encrypted_key"] = None + super(DBInstance, self).__init__(**kwargs) self.set_task_status(task_status) + @property + def key(self): + if self.encrypted_key is None: + return None + + return cu.decrypt_data(cu.decode_data(self.encrypted_key), + CONF.inst_rpc_key_encr_key) + def _validate(self, errors): if InstanceTask.from_code(self.task_id) is None: errors['task_id'] = "Not valid." @@ -1534,6 +1566,56 @@ class DBInstance(dbmodels.DatabaseModelBase): task_status = property(get_task_status, set_task_status) +class instance_encryption_key_cache(object): + def __init__(self, func, lru_cache_size=10): + self._table = {} + self._lru = [] + self._lru_cache_size = lru_cache_size + self._func = func + + def get(self, instance_id): + if instance_id in self._table: + if self._lru.index(instance_id) > 0: + self._lru.remove(instance_id) + self._lru.insert(0, instance_id) + + return self._table[instance_id] + else: + val = self._func(instance_id) + + # BUG(1650518): Cleanup in the Pike release + if val is None: + return val + + if len(self._lru) == self._lru_cache_size: + tail = self._lru.pop() + del self._table[tail] + + self._lru.insert(0, instance_id) + self._table[instance_id] = val + return self._table[instance_id] + + def __getitem__(self, instance_id): + return self.get(instance_id) + + +def _get_instance_encryption_key(instance_id): + instance = DBInstance.find_by(id=instance_id) + + if instance is not None: + return instance.key + else: + raise exception.NotFound(uuid=id) + + +_instance_encryption_key = instance_encryption_key_cache( + func=_get_instance_encryption_key) + + +def get_instance_encryption_key(instance_id): + return _instance_encryption_key[instance_id] + + def persist_instance_fault(notification, event_qualifier): """This callback is registered to be fired whenever a notification is sent out. diff --git a/trove/instance/views.py b/trove/instance/views.py index 6721ec10ce..30c045c794 100644 --- a/trove/instance/views.py +++ b/trove/instance/views.py @@ -127,6 +127,8 @@ class InstanceDetailView(InstanceView): if self.context.is_admin: result['instance']['server_id'] = self.instance.server_id result['instance']['volume_id'] = self.instance.volume_id + result['instance']['encrypted_rpc_messaging'] = ( + self.instance.encrypted_rpc_messaging) return result diff --git a/trove/rpc.py b/trove/rpc.py index 03a63d21c9..dff472ee4b 100644 --- a/trove/rpc.py +++ b/trove/rpc.py @@ -23,7 +23,6 @@ __all__ = [ 'add_extra_exmods', 'clear_extra_exmods', 'get_allowed_exmods', - 'RequestContextSerializer', 'get_client', 'get_server', 'get_notifier', @@ -32,12 +31,10 @@ __all__ = [ from oslo_config import cfg import oslo_messaging as messaging -from oslo_serialization import jsonutils -from osprofiler import profiler -from trove.common.context import TroveContext import trove.common.exception - +from trove.common.rpc import secure_serializer as ssz +from trove.common.rpc import serializer as sz CONF = cfg.CONF TRANSPORT = None @@ -56,7 +53,8 @@ def init(conf): TRANSPORT = messaging.get_transport(conf, allowed_remote_exmods=exmods) - serializer = RequestContextSerializer(JsonPayloadSerializer()) + serializer = sz.TroveRequestContextSerializer( + messaging.JsonPayloadSerializer()) NOTIFIER = messaging.Notifier(TRANSPORT, serializer=serializer) @@ -84,60 +82,26 @@ def get_allowed_exmods(): return ALLOWED_EXMODS + EXTRA_EXMODS -class JsonPayloadSerializer(messaging.NoOpSerializer): - @staticmethod - def serialize_entity(context, entity): - return jsonutils.to_primitive(entity, convert_instances=True) - - -class RequestContextSerializer(messaging.Serializer): - - def __init__(self, base): - self._base = base - - def serialize_entity(self, context, entity): - if not self._base: - return entity - return self._base.serialize_entity(context, entity) - - def deserialize_entity(self, context, entity): - if not self._base: - return entity - return self._base.deserialize_entity(context, entity) - - def serialize_context(self, context): - _context = context.to_dict() - prof = profiler.get() - if prof: - trace_info = { - "hmac_key": prof.hmac_key, - "base_id": prof.get_base_id(), - "parent_id": prof.get_id() - } - _context.update({"trace_info": trace_info}) - return _context - - def deserialize_context(self, context): - trace_info = context.pop("trace_info", None) - if trace_info: - profiler.init(**trace_info) - return TroveContext.from_dict(context) - - def get_transport_url(url_str=None): return messaging.TransportURL.parse(CONF, url_str) -def get_client(target, version_cap=None, serializer=None): +def get_client(target, key, version_cap=None, serializer=None, + secure_serializer=ssz.SecureSerializer): assert TRANSPORT is not None - serializer = RequestContextSerializer(serializer) + # BUG(1650518): Cleanup in the Pike release + # uncomment this (following) line in the pike release + # assert key is not None + serializer = secure_serializer( + sz.TroveRequestContextSerializer(serializer), key) return messaging.RPCClient(TRANSPORT, target, version_cap=version_cap, serializer=serializer) -def get_server(target, endpoints, serializer=None): +def get_server(target, endpoints, key, serializer=None, + secure_serializer=ssz.SecureSerializer): assert TRANSPORT is not None # Thread module is not monkeypatched if remote debugging is enabled. @@ -148,7 +112,12 @@ def get_server(target, endpoints, serializer=None): executor = "blocking" if debug_utils.enabled() else "eventlet" - serializer = RequestContextSerializer(serializer) + # BUG(1650518): Cleanup in the Pike release + # uncomment this (following) line in the pike release + # assert key is not None + serializer = secure_serializer( + sz.TroveRequestContextSerializer(serializer), key) + return messaging.get_rpc_server(TRANSPORT, target, endpoints, diff --git a/trove/taskmanager/api.py b/trove/taskmanager/api.py index 1c1b01aa7f..437e720b21 100644 --- a/trove/taskmanager/api.py +++ b/trove/taskmanager/api.py @@ -77,7 +77,12 @@ class API(object): cctxt.cast(self.context, method_name, **kwargs) def get_client(self, target, version_cap, serializer=None): - return rpc.get_client(target, + if CONF.enable_secure_rpc_messaging: + key = CONF.taskmanager_rpc_encr_key + else: + key = None + + return rpc.get_client(target, key=key, version_cap=version_cap, serializer=serializer) diff --git a/trove/taskmanager/models.py b/trove/taskmanager/models.py index 481996188a..32deb57f73 100755 --- a/trove/taskmanager/models.py +++ b/trove/taskmanager/models.py @@ -31,6 +31,7 @@ from trove.cluster.models import Cluster from trove.cluster.models import DBCluster from trove.cluster import tasks from trove.common import cfg +from trove.common import crypto_utils as cu from trove.common import exception from trove.common.exception import BackupCreationError from trove.common.exception import GuestError @@ -1420,6 +1421,24 @@ class BuiltInstanceTasks(BuiltInstance, NotifyMixin, ConfigurationMixin): volume_device = self._fix_device_path( volume.attachments[0]['device']) + # BUG(1650518): Cleanup in the Pike release some instances + # that we will be upgrading will be pre secureserialier + # and will have no instance_key entries. If this is one of + # those instances, make a key. That will make it appear in + # the injected files that are generated next. From this + # point, and until the guest comes up, attempting to send + # messages to it will fail because the RPC framework will + # encrypt messages to a guest which potentially doesn't + # have the code to handle it. + if CONF.enable_secure_rpc_messaging and ( + self.db_info.encrypted_key is None): + encrypted_key = cu.encode_data(cu.encrypt_data( + cu.generate_random_key(), + CONF.inst_rpc_key_encr_key)) + self.update_db(encrypted_key=encrypted_key) + LOG.debug("Generated unique RPC encryption key for " + "instance = %s, key = %s" % (self.id, encrypted_key)) + injected_files = self.get_injected_files( datastore_version.manager) LOG.debug("Rebuilding instance %(instance)s with image %(image)s.", diff --git a/trove/tests/unittests/common/test_conductor_serializer.py b/trove/tests/unittests/common/test_conductor_serializer.py new file mode 100644 index 0000000000..ae5e5ca0c7 --- /dev/null +++ b/trove/tests/unittests/common/test_conductor_serializer.py @@ -0,0 +1,110 @@ +# Copyright 2016 Tesora, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +import mock + +from trove.common import cfg +from trove.common.rpc import conductor_guest_serializer as gsz +from trove.common.rpc import conductor_host_serializer as hsz + +from trove.tests.unittests import trove_testtools + + +CONF = cfg.CONF + + +class FakeInstance(object): + def __init__(self): + self.uuid = 'a3af1652-686a-4574-a916-2ef7e85136e5' + + @property + def key(self): + return 'mo79Y86Bp3bzQDWR31ihhVGfLBmeac' + + +class FakeContext(object): + def __init__(self, instance_id=None, fields=None): + self.instance_id = instance_id + self.fields = fields + + +class TestConductorSerializer(trove_testtools.TestCase): + + def setUp(self): + self.uuid = 'a3af1652-686a-4574-a916-2ef7e85136e5' + self.key = 'mo79Y86Bp3bzQDWR31ihhVGfLBmeac' + self.data = 'ELzWd81qtgcj2Gxc1ipbh0HgbvHGrgptDj3n4GNMBN0F2WtNdr' + self.context = {'a': 'ij2J8AJLyz0rDqbjxy4jPVINhnK2jsBGpWRKIe3tUnUD', + 'b': 32, + 'c': {'a': 21, 'b': 22}} + self.old_guest_id = gsz.CONF.guest_id + gsz.CONF.guest_id = self.uuid + super(TestConductorSerializer, self).setUp() + + def tearDown(self): + gsz.CONF.guest_id = self.old_guest_id + super(TestConductorSerializer, self).tearDown() + + def test_gsz_serialize_entity_nokey(self): + sz = gsz.ConductorGuestSerializer(None, None) + self.assertEqual(sz.serialize_entity(self.context, self.data), + self.data) + + def test_gsz_serialize_context_nokey(self): + sz = gsz.ConductorGuestSerializer(None, None) + self.assertEqual(sz.serialize_context(self.context), + self.context) + + @mock.patch('trove.common.rpc.conductor_host_serializer.' + 'get_instance_encryption_key', + return_value='mo79Y86Bp3bzQDWR31ihhVGfLBmeac') + def test_hsz_serialize_entity_nokey_noinstance(self, _): + sz = hsz.ConductorHostSerializer(None, None) + ctxt = FakeContext(instance_id=None) + self.assertEqual(sz.serialize_entity(ctxt, self.data), + self.data) + + @mock.patch('trove.common.rpc.conductor_host_serializer.' + 'get_instance_encryption_key', + return_value='mo79Y86Bp3bzQDWR31ihhVGfLBmeac') + def test_hsz_serialize_context_nokey_noinstance(self, _): + sz = hsz.ConductorHostSerializer(None, None) + ctxt = FakeContext(instance_id=None) + self.assertEqual(sz.serialize_context(ctxt), ctxt) + + @mock.patch('trove.common.rpc.conductor_host_serializer.' + 'get_instance_encryption_key', + return_value='mo79Y86Bp3bzQDWR31ihhVGfLBmeac') + def test_conductor_entity(self, _): + guestsz = gsz.ConductorGuestSerializer(None, self.key) + hostsz = hsz.ConductorHostSerializer(None, None) + encrypted_entity = guestsz.serialize_entity(self.context, self.data) + self.assertNotEqual(encrypted_entity, self.data) + entity = hostsz.deserialize_entity(self.context, encrypted_entity) + self.assertEqual(entity, self.data) + + @mock.patch('trove.common.rpc.conductor_host_serializer.' + 'get_instance_encryption_key', + return_value='mo79Y86Bp3bzQDWR31ihhVGfLBmeac') + def test_conductor_context(self, _): + guestsz = gsz.ConductorGuestSerializer(None, self.key) + hostsz = hsz.ConductorHostSerializer(None, None) + encrypted_context = guestsz.serialize_context(self.context) + self.assertNotEqual(encrypted_context, self.context) + context = hostsz.deserialize_context(encrypted_context) + self.assertEqual(context.get('instance_id'), self.uuid) + context.pop('instance_id') + self.assertDictEqual(context, self.context) diff --git a/trove/tests/unittests/common/test_secure_serializer.py b/trove/tests/unittests/common/test_secure_serializer.py new file mode 100644 index 0000000000..2eafe96c75 --- /dev/null +++ b/trove/tests/unittests/common/test_secure_serializer.py @@ -0,0 +1,64 @@ +# Copyright 2016 Tesora, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +from trove.common.rpc import secure_serializer as ssz +from trove.tests.unittests import trove_testtools + + +class TestSecureSerializer(trove_testtools.TestCase): + + def setUp(self): + self.key = 'xuUyAKn5mDANoM5sRxQsb6HGiugWVD' + self.data = '5rzFfaKU630rRxL1g3c80EHnHDf534' + self.context = {'fld1': 3, 'fld2': 'abc'} + super(TestSecureSerializer, self).setUp() + + def tearDown(self): + super(TestSecureSerializer, self).tearDown() + + def test_sz_nokey_serialize_entity(self): + sz = ssz.SecureSerializer(base=None, key=None) + en = sz.serialize_entity(self.context, self.data) + self.assertEqual(en, self.data) + + def test_sz_nokey_deserialize_entity(self): + sz = ssz.SecureSerializer(base=None, key=None) + en = sz.deserialize_entity(self.context, self.data) + self.assertEqual(en, self.data) + + def test_sz_nokey_serialize_context(self): + sz = ssz.SecureSerializer(base=None, key=None) + en = sz.serialize_context(self.context) + self.assertEqual(en, self.context) + + def test_sz_nokey_deserialize_context(self): + sz = ssz.SecureSerializer(base=None, key=None) + en = sz.deserialize_context(self.context) + self.assertEqual(en, self.context) + + def test_sz_entity(self): + sz = ssz.SecureSerializer(base=None, key=self.key) + en = sz.serialize_entity(self.context, self.data) + self.assertNotEqual(en, self.data) + self.assertEqual(sz.deserialize_entity(self.context, en), + self.data) + + def test_sz_context(self): + sz = ssz.SecureSerializer(base=None, key=self.key) + sctxt = sz.serialize_context(self.context) + self.assertNotEqual(sctxt, self.context) + self.assertEqual(sz.deserialize_context(sctxt), + self.context) diff --git a/trove/tests/unittests/common/test_serializer.py b/trove/tests/unittests/common/test_serializer.py new file mode 100644 index 0000000000..ab4696b3f7 --- /dev/null +++ b/trove/tests/unittests/common/test_serializer.py @@ -0,0 +1,127 @@ +# Copyright 2016 Tesora, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +import mock + +from trove.common.rpc import serializer +from trove.tests.unittests import trove_testtools + + +class TestSerializer(trove_testtools.TestCase): + + def setUp(self): + self.data = 'abcdefghijklmnopqrstuvwxyz' + self.context = {} + super(TestSerializer, self).setUp() + + def tearDown(self): + super(TestSerializer, self).tearDown() + + def test_serialize_1(self): + base = mock.Mock() + sz = serializer.TroveSerializer(base=base) + sz.serialize_entity(self.context, self.data) + base.serialize_entity.assert_called_with(self.context, self.data) + + def test_serialize_2(self): + base = mock.Mock() + sz1 = serializer.TroveSerializer(base=base) + sz = serializer.TroveSerializer(base=sz1) + sz.serialize_entity(self.context, self.data) + base.serialize_entity.assert_called_with(self.context, self.data) + + def test_serialize_3(self): + base = mock.Mock() + sz = serializer.TroveSerializer(base=base) + sz.deserialize_entity(self.context, self.data) + base.deserialize_entity.assert_called_with(self.context, self.data) + + def test_serialize_4(self): + base = mock.Mock() + sz1 = serializer.TroveSerializer(base=base) + sz = serializer.TroveSerializer(base=sz1) + sz.deserialize_entity(self.context, self.data) + base.deserialize_entity.assert_called_with(self.context, self.data) + + def test_serialize_5(self): + base = mock.Mock() + sz = serializer.TroveSerializer(base=base) + sz.serialize_context(self.context) + base.serialize_context.assert_called_with(self.context) + + def test_serialize_6(self): + base = mock.Mock() + sz1 = serializer.TroveSerializer(base=base) + sz = serializer.TroveSerializer(base=sz1) + sz.serialize_context(self.context) + base.serialize_context.assert_called_with(self.context) + + def test_serialize_7(self): + base = mock.Mock() + sz = serializer.TroveSerializer(base=base) + sz.deserialize_context(self.context) + base.deserialize_context.assert_called_with(self.context) + + def test_serialize_8(self): + base = mock.Mock() + sz1 = serializer.TroveSerializer(base=base) + sz = serializer.TroveSerializer(base=sz1) + sz.deserialize_context(self.context) + base.deserialize_context.assert_called_with(self.context) + + def test_serialize_9(self): + sz = serializer.TroveSerializer(base=None) + self.assertEqual(sz.serialize_entity(self.context, self.data), + self.data) + + def test_serialize_10(self): + sz = serializer.TroveSerializer(base=None) + self.assertEqual(sz.deserialize_entity(self.context, self.data), + self.data) + + def test_serialize_11(self): + sz = serializer.TroveSerializer(base=None) + self.assertEqual(sz.serialize_context(self.context), + self.context) + + def test_serialize_12(self): + sz = serializer.TroveSerializer(base=None) + self.assertEqual(sz.deserialize_context(self.context), + self.context) + + def test_serialize_13(self): + bz = serializer.TroveSerializer(base=None) + sz = serializer.TroveSerializer(base=bz) + self.assertEqual(sz.serialize_entity(self.context, self.data), + self.data) + + def test_serialize_14(self): + bz = serializer.TroveSerializer(base=None) + sz = serializer.TroveSerializer(base=bz) + self.assertEqual(sz.deserialize_entity(self.context, self.data), + self.data) + + def test_serialize_15(self): + bz = serializer.TroveSerializer(base=None) + sz = serializer.TroveSerializer(base=bz) + self.assertEqual(sz.serialize_context(self.context), + self.context) + + def test_serialize_16(self): + bz = serializer.TroveSerializer(base=None) + sz = serializer.TroveSerializer(base=bz) + self.assertEqual(sz.deserialize_context(self.context), + self.context) diff --git a/trove/tests/unittests/conductor/test_conf.py b/trove/tests/unittests/conductor/test_conf.py index c4305bbe1b..924dc693f7 100644 --- a/trove/tests/unittests/conductor/test_conf.py +++ b/trove/tests/unittests/conductor/test_conf.py @@ -32,7 +32,8 @@ def mocked_conf(manager): 'conductor_manager': manager, 'trove_conductor_workers': 1, 'host': 'mockhost', - 'report_interval': 1}) + 'report_interval': 1, + 'instance_rpc_encr_key': ''}) class NoopManager(object): diff --git a/trove/tests/unittests/guestagent/test_api.py b/trove/tests/unittests/guestagent/test_api.py index 5390fbea75..71efb963f8 100644 --- a/trove/tests/unittests/guestagent/test_api.py +++ b/trove/tests/unittests/guestagent/test_api.py @@ -50,7 +50,9 @@ def _mock_call(cmd, timeout, version=None, username=None, hostname=None, class ApiTest(trove_testtools.TestCase): @mock.patch.object(rpc, 'get_client') - def setUp(self, *args): + @mock.patch('trove.instance.models.get_instance_encryption_key', + return_value='2LMDgren5citVxmSYNiRFCyFfVDjJtDaQT9LYV08') + def setUp(self, mock_get_encryption_key, *args): super(ApiTest, self).setUp() self.context = context.TroveContext() self.guest = api.API(self.context, 0) @@ -58,6 +60,7 @@ class ApiTest(trove_testtools.TestCase): self.guest._call = _mock_call self.api = api.API(self.context, "instance-id-x23d2d") self._mock_rpc_client() + mock_get_encryption_key.assert_called() def test_change_passwords(self): self.assertIsNone(self.guest.change_passwords("dummy")) diff --git a/trove/tests/unittests/guestagent/test_galera_cluster_api.py b/trove/tests/unittests/guestagent/test_galera_cluster_api.py index 9f79eb5664..809d9e1abe 100644 --- a/trove/tests/unittests/guestagent/test_galera_cluster_api.py +++ b/trove/tests/unittests/guestagent/test_galera_cluster_api.py @@ -37,7 +37,9 @@ def _mock_call(cmd, timeout, version=None, user=None, class ApiTest(trove_testtools.TestCase): @mock.patch.object(rpc, 'get_client') - def setUp(self, *args): + @mock.patch('trove.instance.models.get_instance_encryption_key', + return_value='2LMDgren5citVxmSYNiRFCyFfVDjJtDaQT9LYV08') + def setUp(self, mock_get_encryption_key, *args): super(ApiTest, self).setUp() cluster_guest_api = (GaleraCommonGuestAgentStrategy() .guest_client_class) @@ -46,6 +48,7 @@ class ApiTest(trove_testtools.TestCase): self.guest._call = _mock_call self.api = cluster_guest_api(self.context, "instance-id-x23d2d") self._mock_rpc_client() + mock_get_encryption_key.assert_called() def test_get_routing_key(self): self.assertEqual('guestagent.instance-id-x23d2d', diff --git a/trove/tests/unittests/guestagent/test_vertica_api.py b/trove/tests/unittests/guestagent/test_vertica_api.py index b67b9e5f18..7c47cc3dbd 100644 --- a/trove/tests/unittests/guestagent/test_vertica_api.py +++ b/trove/tests/unittests/guestagent/test_vertica_api.py @@ -37,13 +37,17 @@ def _mock_call(cmd, timeout, version=None, user=None, class ApiTest(trove_testtools.TestCase): @mock.patch.object(rpc, 'get_client') - def setUp(self, *args): + @mock.patch('trove.instance.models.get_instance_encryption_key', + return_value='2LMDgren5citVxmSYNiRFCyFfVDjJtDaQT9LYV08') + def setUp(self, mock_get_encryption_key, *args): super(ApiTest, self).setUp() self.context = context.TroveContext() self.guest = VerticaGuestAgentAPI(self.context, 0) + self.guest._call = _mock_call self.api = VerticaGuestAgentAPI(self.context, "instance-id-x23d2d") self._mock_rpc_client() + mock_get_encryption_key.assert_called() def test_get_routing_key(self): self.assertEqual('guestagent.instance-id-x23d2d', diff --git a/trove/tests/unittests/instance/test_instance_models.py b/trove/tests/unittests/instance/test_instance_models.py index c60120be18..c089daaad4 100644 --- a/trove/tests/unittests/instance/test_instance_models.py +++ b/trove/tests/unittests/instance/test_instance_models.py @@ -26,6 +26,7 @@ from trove.instance.models import DBInstance from trove.instance.models import DBInstanceFault from trove.instance.models import filter_ips from trove.instance.models import Instance +from trove.instance.models import instance_encryption_key_cache from trove.instance.models import InstanceServiceStatus from trove.instance.models import SimpleInstance from trove.instance.tasks import InstanceTasks @@ -469,3 +470,53 @@ class TestModules(trove_testtools.TestCase): expected_exception, models.validate_modules_for_apply, modules, ds_id, ds_ver_id) + + +def trivial_key_function(id): + return id * id + + +class TestInstanceKeyCaching(trove_testtools.TestCase): + + def setUp(self): + super(TestInstanceKeyCaching, self).setUp() + + def tearDown(self): + super(TestInstanceKeyCaching, self).tearDown() + + def test_basic_caching(self): + keycache = instance_encryption_key_cache(trivial_key_function, 5) + self.assertEqual(keycache[5], 25) + self.assertEqual(keycache[5], 25) + self.assertEqual(keycache[25], 625) + + def test_caching(self): + keyfn = Mock(return_value=123) + keycache = instance_encryption_key_cache(keyfn, 5) + self.assertEqual(keycache[5], 123) + self.assertEqual(keyfn.call_count, 1) + self.assertEqual(keycache[5], 123) + self.assertEqual(keyfn.call_count, 1) + self.assertEqual(keycache[6], 123) + self.assertEqual(keyfn.call_count, 2) + self.assertEqual(keycache[7], 123) + self.assertEqual(keyfn.call_count, 3) + self.assertEqual(keycache[8], 123) + self.assertEqual(keyfn.call_count, 4) + self.assertEqual(keycache[9], 123) + self.assertEqual(keyfn.call_count, 5) + self.assertEqual(keycache[10], 123) + self.assertEqual(keyfn.call_count, 6) + self.assertEqual(keycache[10], 123) + self.assertEqual(keyfn.call_count, 6) + self.assertEqual(keycache[5], 123) + self.assertEqual(keyfn.call_count, 7) + + # BUG(1650518): Cleanup in the Pike release + def test_not_caching_none(self): + keyfn = Mock(return_value=None) + keycache = instance_encryption_key_cache(keyfn, 5) + self.assertIsNone(keycache[30]) + self.assertEqual(keyfn.call_count, 1) + self.assertIsNone(keycache[30]) + self.assertEqual(keyfn.call_count, 2) diff --git a/trove/tests/unittests/taskmanager/test_models.py b/trove/tests/unittests/taskmanager/test_models.py index 4729719917..1140eaf3cb 100644 --- a/trove/tests/unittests/taskmanager/test_models.py +++ b/trove/tests/unittests/taskmanager/test_models.py @@ -245,7 +245,8 @@ class FreshInstanceTasksTest(trove_testtools.TestCase): None, None, None, datastore_manager, None, None, None) self.assertEqual(server.userdata, self.userdata) - def test_create_instance_guestconfig(self): + @patch.object(DBInstance, 'get_by') + def test_create_instance_guestconfig(self, patch_get_by): def fake_conf_getter(*args, **kwargs): if args[0] == 'guest_config': return self.guestconfig @@ -268,7 +269,8 @@ class FreshInstanceTasksTest(trove_testtools.TestCase): self.guestconfig_content, files['/etc/trove/conf.d/trove-guestagent.conf']) - def test_create_instance_guestconfig_compat(self): + @patch.object(DBInstance, 'get_by') + def test_create_instance_guestconfig_compat(self, patch_get_by): def fake_conf_getter(*args, **kwargs): if args[0] == 'guest_config': return self.guestconfig @@ -460,7 +462,8 @@ class FreshInstanceTasksTest(trove_testtools.TestCase): @patch.object(trove.guestagent.api.API, 'attach_replication_slave') @patch.object(rpc, 'get_client') - def test_attach_replication_slave(self, mock_get_client, + @patch.object(DBInstance, 'get_by') + def test_attach_replication_slave(self, mock_get_by, mock_get_client, mock_attach_replication_slave): mock_flavor = {'id': 8, 'ram': 768, 'name': 'bigger_flavor'} snapshot = {'replication_strategy': 'MysqlGTIDReplication', @@ -483,6 +486,7 @@ class FreshInstanceTasksTest(trove_testtools.TestCase): @patch.object(trove.guestagent.api.API, 'attach_replication_slave', side_effect=GuestError) @patch('trove.taskmanager.models.LOG') + @patch.object(DBInstance, 'get_by') def test_error_attach_replication_slave(self, *args): mock_flavor = {'id': 8, 'ram': 768, 'name': 'bigger_flavor'} snapshot = {'replication_strategy': 'MysqlGTIDReplication', diff --git a/trove/tests/unittests/upgrade/test_models.py b/trove/tests/unittests/upgrade/test_models.py index f04594627a..9a859de6a5 100644 --- a/trove/tests/unittests/upgrade/test_models.py +++ b/trove/tests/unittests/upgrade/test_models.py @@ -66,7 +66,11 @@ class TestUpgradeModel(trove_testtools.TestCase): @patch('trove.guestagent.api.API.upgrade') @patch.object(rpc, 'get_client') - def _assert_create_with_metadata(self, mock_client, api_upgrade_mock, + @patch('trove.instance.models.get_instance_encryption_key', + return_value='2LMDgren5citVxmSYNiRFCyFfVDjJtDaQT9LYV08') + def _assert_create_with_metadata(self, mock_get_encryption_key, + mock_client, + api_upgrade_mock, metadata=None): """Exercise UpgradeMessageSender.create() call. """ @@ -85,3 +89,4 @@ class TestUpgradeModel(trove_testtools.TestCase): func() # This call should translate to the API call asserted below. api_upgrade_mock.assert_called_once_with(instance_version, location, metadata) + mock_get_encryption_key.assert_called()