Add Google Cloud Engine metadata service
Add cloudbaseinit.metadata.services.gceservice.GCEService that supports Google Cloud Engine. Supported features for the metadata service: * instance id * hostname * userdata * ssh keys Change-Id: I9e5e2cbcaa0953fc0c0ae8117e258713ac6443b7
This commit is contained in:
Adrian Vladu
parent
a77477e16e
commit
29105932c0
|
|
@ -25,6 +25,7 @@ _OPT_PATHS = (
|
|||
'cloudbaseinit.conf.ovf.OvfOptions',
|
||||
'cloudbaseinit.conf.packet.PacketOptions',
|
||||
'cloudbaseinit.conf.vmwareguestinfo.VMwareGuestInfoConfigOptions',
|
||||
'cloudbaseinit.conf.gce.GCEOptions',
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,51 @@
|
|||
# Copyright 2020 Cloudbase Solutions Srl
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Config options available for the GCE metadata service."""
|
||||
|
||||
from oslo_config import cfg
|
||||
|
||||
from cloudbaseinit.conf import base as conf_base
|
||||
|
||||
|
||||
class GCEOptions(conf_base.Options):
|
||||
|
||||
"""Config options available for the GCE metadata service."""
|
||||
|
||||
def __init__(self, config):
|
||||
super(GCEOptions, self).__init__(config, group="gce")
|
||||
self._options = [
|
||||
cfg.StrOpt(
|
||||
"metadata_base_url",
|
||||
default="http://metadata.google.internal/computeMetadata/v1/",
|
||||
help="The base URL where the service looks for metadata"),
|
||||
cfg.BoolOpt(
|
||||
"https_allow_insecure", default=False,
|
||||
help="Whether to disable the validation of HTTPS "
|
||||
"certificates."),
|
||||
cfg.StrOpt(
|
||||
"https_ca_bundle", default=None,
|
||||
help="The path to a CA_BUNDLE file or directory with "
|
||||
"certificates of trusted CAs."),
|
||||
]
|
||||
|
||||
def register(self):
|
||||
"""Register the current options to the global ConfigOpts object."""
|
||||
group = cfg.OptGroup(self.group_name, title='GCE Options')
|
||||
self._config.register_group(group)
|
||||
self._config.register_opts(self._options, group=group)
|
||||
|
||||
def list(self):
|
||||
"""Return a list which contains all the available options."""
|
||||
return self._options
|
||||
|
|
@ -0,0 +1,184 @@
|
|||
# Copyright 2020 Cloudbase Solutions Srl
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import base64
|
||||
import json
|
||||
|
||||
from datetime import datetime
|
||||
from oslo_log import log as oslo_logging
|
||||
|
||||
from cloudbaseinit import conf as cloudbaseinit_conf
|
||||
from cloudbaseinit.metadata.services import base
|
||||
|
||||
CONF = cloudbaseinit_conf.CONF
|
||||
LOG = oslo_logging.getLogger(__name__)
|
||||
|
||||
GCE_METADATA_HEADERS = {'Metadata-Flavor': 'Google'}
|
||||
MD_INSTANCE = "instance"
|
||||
MD_INSTANCE_ATTR = "%s/attributes" % MD_INSTANCE
|
||||
MD_PROJECT_ATTR = "project/attributes"
|
||||
|
||||
|
||||
class GCEService(base.BaseHTTPMetadataService):
|
||||
|
||||
def __init__(self):
|
||||
super(GCEService, self).__init__(
|
||||
base_url=CONF.gce.metadata_base_url,
|
||||
https_allow_insecure=CONF.gce.https_allow_insecure,
|
||||
https_ca_bundle=CONF.gce.https_ca_bundle)
|
||||
self._enable_retry = True
|
||||
|
||||
def _http_request(self, url, data=None, headers=None, method=None):
|
||||
headers = headers or {}
|
||||
headers.update(GCE_METADATA_HEADERS)
|
||||
|
||||
return super(GCEService, self)._http_request(url, data,
|
||||
headers, method)
|
||||
|
||||
def load(self):
|
||||
super(GCEService, self).load()
|
||||
|
||||
try:
|
||||
self.get_host_name()
|
||||
return True
|
||||
except base.NotExistingMetadataException:
|
||||
LOG.debug("Metadata not found at URL '%s'",
|
||||
CONF.gce.metadata_base_url)
|
||||
|
||||
def get_host_name(self):
|
||||
return self._get_cache_data('%s/name' % MD_INSTANCE, decode=True)
|
||||
|
||||
def get_instance_id(self):
|
||||
return self._get_cache_data('%s/id' % MD_INSTANCE, decode=True)
|
||||
|
||||
def get_user_data(self):
|
||||
user_data = self._get_cache_data('%s/user-data' % MD_INSTANCE_ATTR)
|
||||
try:
|
||||
encoding = self._get_cache_data(
|
||||
'%s/user-data-encoding' % MD_INSTANCE_ATTR,
|
||||
decode=True)
|
||||
if encoding:
|
||||
if encoding == 'base64':
|
||||
user_data = base64.b64decode(user_data)
|
||||
else:
|
||||
LOG.warning("Encoding '%s' not supported. "
|
||||
"Falling back to plaintext", encoding)
|
||||
except base.NotExistingMetadataException:
|
||||
LOG.info('Userdata encoding could not be found in the metadata.')
|
||||
|
||||
return user_data
|
||||
|
||||
def _is_ssh_key_valid(self, expire_on):
|
||||
if not expire_on:
|
||||
return True
|
||||
try:
|
||||
time_format = '%Y-%m-%dT%H:%M:%S+0000'
|
||||
expire_time = datetime.strptime(expire_on, time_format)
|
||||
return datetime.utcnow() <= expire_time
|
||||
except ValueError:
|
||||
# Note(ader1990): Return True to be consistent with cloud-init
|
||||
return True
|
||||
|
||||
def _parse_gce_ssh_key(self, raw_ssh_key):
|
||||
# GCE public keys have a special format defined here:
|
||||
# https://cloud.google.com/compute/docs/instances/adding-removing-ssh-keys#sshkeyformat
|
||||
INVALID_SSH_KEY_MSG = "Skipping invalid SSH key %s"
|
||||
header_username = None
|
||||
meta_username = None
|
||||
|
||||
if not raw_ssh_key:
|
||||
return
|
||||
|
||||
ssh_key = raw_ssh_key.strip()
|
||||
|
||||
# Key is in the format: USERNAME:ssh-rsa ...
|
||||
# Remove the username from the key
|
||||
ssh_key_split = ssh_key.split(':', 1)
|
||||
if len(ssh_key_split) != 2:
|
||||
LOG.warning(INVALID_SSH_KEY_MSG, ssh_key)
|
||||
return
|
||||
|
||||
header_username, ssh_key = ssh_key_split
|
||||
|
||||
key_parts = ssh_key.split(' ')
|
||||
len_key_parts = len(key_parts)
|
||||
|
||||
if len_key_parts < 3:
|
||||
# Key format not supported: USERNAME:ssh-rsa [KEY]
|
||||
LOG.warning(INVALID_SSH_KEY_MSG, ssh_key)
|
||||
return
|
||||
elif len_key_parts == 3:
|
||||
# Key format: USERNAME:ssh-rsa [KEY] [USERNAME]
|
||||
meta_username = key_parts[2]
|
||||
else:
|
||||
# Key format: USERNAME:ssh-rsa [KEY] google-ssh [JSON_METADATA]
|
||||
delimiter = 'google-ssh'
|
||||
json_key_parts = ssh_key.split(delimiter)
|
||||
if (len(json_key_parts) == 2 and json_key_parts[1]):
|
||||
ssh_key_metadata = json.loads(json_key_parts[1].strip())
|
||||
meta_username = ssh_key_metadata['userName']
|
||||
if not self._is_ssh_key_valid(ssh_key_metadata['expireOn']):
|
||||
LOG.warning("Skipping expired key: %s", ssh_key)
|
||||
return
|
||||
ssh_key = '%s %s' % (json_key_parts[0].strip(), meta_username)
|
||||
else:
|
||||
LOG.warning(INVALID_SSH_KEY_MSG, ssh_key)
|
||||
return
|
||||
|
||||
if not (header_username == meta_username == CONF.username):
|
||||
LOG.warning("Skipping key due to non matching username: %s",
|
||||
ssh_key)
|
||||
return
|
||||
|
||||
return ssh_key
|
||||
|
||||
def _get_ssh_keys(self, locations):
|
||||
ssh_keys = []
|
||||
for location in locations:
|
||||
try:
|
||||
raw_ssh_keys = self._get_cache_data(location, decode=True)
|
||||
ssh_keys += raw_ssh_keys.strip().splitlines()
|
||||
except base.NotExistingMetadataException:
|
||||
LOG.warning("SSH keys not found at location %s", location)
|
||||
|
||||
return ssh_keys
|
||||
|
||||
def get_public_keys(self):
|
||||
ssh_keys = []
|
||||
raw_ssh_keys = []
|
||||
block_project_keys = False
|
||||
key_locations = ["%s/ssh-keys" % MD_INSTANCE_ATTR]
|
||||
|
||||
# Use GCE latest metadata, where the SSH keys are found
|
||||
# only at hyphenated locations
|
||||
try:
|
||||
block_key = "%s/block-project-ssh-keys" % MD_INSTANCE_ATTR
|
||||
if self._get_cache_data(block_key, decode=True) == 'true':
|
||||
block_project_keys = True
|
||||
except base.NotExistingMetadataException:
|
||||
LOG.debug('block-project-ssh-keys not present')
|
||||
|
||||
if not block_project_keys:
|
||||
key_locations += [
|
||||
"%s/ssh-keys" % MD_PROJECT_ATTR
|
||||
]
|
||||
|
||||
raw_ssh_keys += self._get_ssh_keys(key_locations)
|
||||
|
||||
for raw_ssh_key in raw_ssh_keys:
|
||||
ssh_key = self._parse_gce_ssh_key(raw_ssh_key)
|
||||
if ssh_key:
|
||||
ssh_keys.append(ssh_key)
|
||||
|
||||
return ssh_keys
|
||||
|
|
@ -0,0 +1,172 @@
|
|||
# Copyright 2020 Cloudbase Solutions Srl
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import ddt
|
||||
import importlib
|
||||
import unittest
|
||||
|
||||
try:
|
||||
import unittest.mock as mock
|
||||
except ImportError:
|
||||
import mock
|
||||
|
||||
from cloudbaseinit import conf as cloudbaseinit_conf
|
||||
from cloudbaseinit.tests import testutils
|
||||
|
||||
|
||||
CONF = cloudbaseinit_conf.CONF
|
||||
BASE_MODULE_PATH = ("cloudbaseinit.metadata.services.base."
|
||||
"BaseHTTPMetadataService")
|
||||
MODULE_PATH = "cloudbaseinit.metadata.services.gceservice"
|
||||
|
||||
|
||||
@ddt.ddt
|
||||
class GCEServiceTest(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self._win32com_mock = mock.MagicMock()
|
||||
self._ctypes_mock = mock.MagicMock()
|
||||
self._ctypes_util_mock = mock.MagicMock()
|
||||
self._win32com_client_mock = mock.MagicMock()
|
||||
self._pywintypes_mock = mock.MagicMock()
|
||||
self._module_patcher = mock.patch.dict(
|
||||
'sys.modules',
|
||||
{'win32com': self._win32com_mock,
|
||||
'ctypes': self._ctypes_mock,
|
||||
'ctypes.util': self._ctypes_util_mock,
|
||||
'win32com.client': self._win32com_client_mock,
|
||||
'pywintypes': self._pywintypes_mock})
|
||||
self._module_patcher.start()
|
||||
self.addCleanup(self._module_patcher.stop)
|
||||
|
||||
self._module = importlib.import_module(MODULE_PATH)
|
||||
self._service = self._module.GCEService()
|
||||
self.snatcher = testutils.LogSnatcher(MODULE_PATH)
|
||||
|
||||
@mock.patch(MODULE_PATH + ".GCEService._get_cache_data")
|
||||
def test_get_host_name(self, mock_get_cache_data):
|
||||
response = self._service.get_host_name()
|
||||
mock_get_cache_data.assert_called_once_with(
|
||||
'instance/name', decode=True)
|
||||
self.assertEqual(mock_get_cache_data.return_value,
|
||||
response)
|
||||
|
||||
@mock.patch(MODULE_PATH + ".GCEService._get_cache_data")
|
||||
def test_get_instance_id(self, mock_get_cache_data):
|
||||
response = self._service.get_instance_id()
|
||||
mock_get_cache_data.assert_called_once_with(
|
||||
'instance/id', decode=True)
|
||||
self.assertEqual(mock_get_cache_data.return_value,
|
||||
response)
|
||||
|
||||
@mock.patch(MODULE_PATH + ".GCEService._get_cache_data")
|
||||
def test_get_user_data(self, mock_get_cache_data):
|
||||
response = self._service.get_user_data()
|
||||
userdata_key = "%s/user-data" % self._module.MD_INSTANCE_ATTR
|
||||
userdata_enc_key = (
|
||||
"%s/user-data-encoding" % self._module.MD_INSTANCE_ATTR)
|
||||
mock_calls = [mock.call(userdata_key),
|
||||
mock.call(userdata_enc_key, decode=True)]
|
||||
mock_get_cache_data.assert_has_calls(mock_calls)
|
||||
self.assertEqual(mock_get_cache_data.return_value,
|
||||
response)
|
||||
|
||||
@mock.patch(MODULE_PATH + ".GCEService._get_cache_data")
|
||||
def test_get_user_data_b64(self, mock_get_cache_data):
|
||||
user_data = b'fake userdata'
|
||||
user_data_b64 = 'ZmFrZSB1c2VyZGF0YQ=='
|
||||
userdata_key = "%s/user-data" % self._module.MD_INSTANCE_ATTR
|
||||
userdata_enc_key = (
|
||||
"%s/user-data-encoding" % self._module.MD_INSTANCE_ATTR)
|
||||
|
||||
def _get_cache_data_side_effect(*args, **kwargs):
|
||||
if args[0] == ("%s/user-data" % self._module.MD_INSTANCE_ATTR):
|
||||
return user_data_b64
|
||||
return 'base64'
|
||||
mock_get_cache_data.side_effect = _get_cache_data_side_effect
|
||||
|
||||
response = self._service.get_user_data()
|
||||
|
||||
mock_calls = [mock.call(userdata_key),
|
||||
mock.call(userdata_enc_key, decode=True)]
|
||||
mock_get_cache_data.assert_has_calls(mock_calls)
|
||||
self.assertEqual(response, user_data)
|
||||
|
||||
@mock.patch(MODULE_PATH + ".GCEService._get_cache_data")
|
||||
@mock.patch(MODULE_PATH + ".GCEService._get_ssh_keys")
|
||||
@mock.patch(MODULE_PATH + ".GCEService._parse_gce_ssh_key")
|
||||
def _test_get_public_keys_block_project(self, mock_parse_keys,
|
||||
mock_get_ssh_keys,
|
||||
mock_cache_data,
|
||||
cache_data_result=False):
|
||||
expected_response = []
|
||||
|
||||
if cache_data_result:
|
||||
second_call_get_ssh = [
|
||||
'%s/ssh-keys' % self._module.MD_INSTANCE_ATTR,
|
||||
'%s/ssh-keys' % self._module.MD_PROJECT_ATTR]
|
||||
mock_cache_data.return_value = 'false'
|
||||
else:
|
||||
second_call_get_ssh = [
|
||||
'%s/ssh-keys' % self._module.MD_INSTANCE_ATTR]
|
||||
mock_cache_data.return_value = 'true'
|
||||
mock_get_ssh_keys.return_value = []
|
||||
response = self._service.get_public_keys()
|
||||
mock_calls = [mock.call(second_call_get_ssh)]
|
||||
mock_get_ssh_keys.assert_has_calls(mock_calls)
|
||||
|
||||
self.assertEqual(mock_parse_keys.call_count, 0)
|
||||
self.assertEqual(response, expected_response)
|
||||
|
||||
def test_get_public_keys_block_project_check(self):
|
||||
self._test_get_public_keys_block_project(cache_data_result=False)
|
||||
|
||||
def test_get_public_keys_block_project(self):
|
||||
self._test_get_public_keys_block_project(cache_data_result=True)
|
||||
|
||||
@mock.patch(MODULE_PATH + ".GCEService._get_cache_data")
|
||||
def test__get_ssh_keys(self, mock_get_cache_data):
|
||||
fake_key = 'fake key'
|
||||
expected_response = [fake_key] * 3
|
||||
key_locations = ['location'] * 3
|
||||
mock_get_cache_data.return_value = fake_key
|
||||
response = self._service._get_ssh_keys(key_locations)
|
||||
self.assertEqual(response, expected_response)
|
||||
|
||||
@ddt.data((None, True),
|
||||
('not a date', True),
|
||||
('2018-12-04T20:12:00+0000', False))
|
||||
@ddt.unpack
|
||||
def test__is_ssh_key_valid(self, expire_on, expected_response):
|
||||
response = self._service._is_ssh_key_valid(expire_on)
|
||||
self.assertEqual(response, expected_response)
|
||||
|
||||
@ddt.data((None, None),
|
||||
('ssh invalid', None),
|
||||
('notadmin:ssh key notadmin', None),
|
||||
('Admin:ssh key Admin', 'ssh key Admin'),
|
||||
('ssh key google-ssh', None),
|
||||
('Admin:s k google-ssh {"userName":"Admin",'
|
||||
'"expireOn":"1018-12-04T20:12:00+0000"}',
|
||||
None),
|
||||
('Admin:s k google-ssh {"userName":"b",'
|
||||
'"expireOn":"3018-12-04T20:12:00+0000"}',
|
||||
None),
|
||||
('Admin:s k google-ssh {"userName":"Admin",'
|
||||
'"expireOn":"3018-12-04T20:12:00+0000"}',
|
||||
's k Admin'))
|
||||
@ddt.unpack
|
||||
def test__parse_gce_ssh_key(self, raw_ssh_key, expected_response):
|
||||
response = self._service._parse_gce_ssh_key(raw_ssh_key)
|
||||
self.assertEqual(response, expected_response)
|
||||
|
|
@ -433,3 +433,37 @@ Capabilities:
|
|||
Config options for `vmwareguestinfo` section:
|
||||
|
||||
* vmware_rpctool_path (string: "%ProgramFiles%/VMware/VMware Tools/rpctool.exe")
|
||||
|
||||
|
||||
Google Compute Engine Service
|
||||
-----------------------------
|
||||
|
||||
.. class:: cloudbaseinit.metadata.services.gceservice.GCEService
|
||||
|
||||
`GCE <https://cloud.google.com/compute/>`_ metadata service provides
|
||||
the metadata for instances running on Google Compute Engine.
|
||||
|
||||
GCE metadata is offered via an internal HTTP metadata endpoint, reachable at the magic URL
|
||||
`http://metadata.google.internal/computeMetadata/v1/`. More information can be found in the GCE
|
||||
metadata `documents <https://cloud.google.com/compute/docs/storing-retrieving-metadata#querying>`_.
|
||||
|
||||
To provide userdata to be executed by the instance (in cloud-config format, for example), use the
|
||||
user-data and user-data-encoding instance metadata keys.
|
||||
|
||||
Capabilities:
|
||||
|
||||
* instance id
|
||||
* hostname
|
||||
* public keys
|
||||
* user data
|
||||
|
||||
Config options for `gce` section:
|
||||
|
||||
* metadata_base_url (string: http://metadata.google.internal/computeMetadata/v1/")
|
||||
* https_allow_insecure (bool: False)
|
||||
* https_ca_bundle (string: None)
|
||||
|
||||
Config options for `default` section:
|
||||
|
||||
* retry_count (integer: 5)
|
||||
* retry_count_interval (integer: 4)
|
||||
|
|
|
|||
Loading…
Reference in New Issue