Execute service setup against a delegated host using Ansible built-in modules

In order to reduce the packages required to pip install on to the hosts,
we allow the service setup to be delegated to a specific host, defaulting
to the deploy host. We also switch as many tasks as possible to using the
built-in Ansible modules which make use of the shade library.

The 'virtualenv' package is now installed appropriately by the openstack_hosts
role, so there's no need to install it any more. The 'httplib2' package is a
legacy Ansible requirement for the get_url/get_uri module which is no longer
needed. The keystone client library is not required any more now that we're
using the upstream modules. As there are no required packages left, the task
to install them is also removed.

With the dependent patches, the openstack_openrc role is now executed once
on the designated host, so it is no longer required as a meta-dependency for
the role.

Depends-On: https://review.openstack.org/579233
Depends-On: https://review.openstack.org/579959
Change-Id: I4131312eea8c743e7803ccc622b7642c6082a4c8

defaults/main.yml

@@ -17,6 +17,11 @@
 # Only create Gnocchi's identity entities in Keystone
 gnocchi_identity_only: False
 
+# Set the host which will execute the shade modules
+# for the service setup. The host must already have
+# clouds.yaml properly configured.
+gnocchi_service_setup_host: "{{ openstack_service_setup_host | default('localhost') }}"
+
 #: Enable for debug logging level
 debug: false
 
@@ -71,7 +76,6 @@ gnocchi_db_setup_host: "{{ ('galera_all' in groups) | ternary(groups['galera_all
 gnocchi_galera_address: "{{ galera_address | default('127.0.0.1') }}"
 gnocchi_galera_database: gnocchi
 gnocchi_galera_user: gnocchi
-gnocchi_galera_address: "{{ galera_address }}"
 gnocchi_db_sync_options: ""
 gnocchi_galera_use_ssl: "{{ galera_use_ssl | default(False) }}"
 gnocchi_galera_ssl_ca_cert: "{{ galera_ssl_ca_cert | default('/etc/ssl/certs/galera-ca.pem') }}"
@@ -159,12 +163,6 @@ gnocchi_services:
     service_enabled: true
     init_config_overrides: "{{ gnocchi_metricd_init_overrides }}"
 
-#: Gnocchi packages that must be installed before anything else
-gnocchi_requires_pip_packages:
-  - virtualenv
-  - python-keystoneclient # Keystoneclient needed for OSA keystone lib
-  - httplib2 # so we can use the uri module
-
 #: Common pip packages
 gnocchi_pip_packages:
   - cryptography

meta/main.yml

@@ -39,6 +39,5 @@ dependencies:
     when:
       - ansible_pkg_mgr == 'apt'
   - galera_client
-  - openstack_openrc
 # Extra dependency not installable this way
 #  git clone https://git.openstack.org/openstack/openstack-ansible-plugins {homedir}/.ansible/plugins

releasenotes/notes/gnocchi-service-setup-host-ef418b0e709ae796.yaml

@@ -0,0 +1,17 @@
+---
+features:
+  - |
+    The service setup in keystone for gnocchi will now be executed
+    through delegation to the ``gnocchi_service_setup_host`` which,
+    by default, is ``localhost`` (the deploy host). Deployers can
+    opt to rather change this to the utility container by implementing
+    the following override in ``user_variables.yml``.
+
+    .. code-block:: yaml
+
+      gnocchi_service_setup_host: "{{ groups['utility_all'][0] }}"
+
+deprecations:
+  - |
+    The variable ``gnocchi_requires_pip_packages`` is no longer required
+    and has therefore been removed.

tasks/gnocchi_identity_setup.yml

@@ -16,54 +16,56 @@
 # Create the project if needed, assumed to be in default domain.
 # In many cases this will be present but under some circumstances the project
 # may be unique to Gnocchi, esp. when Swift is used for storage.
-- name: Ensure Gnocchi project
-  keystone:
-    command: ensure_project
-    project_name: "{{ gnocchi_service_project_name }}"
-    endpoint: "{{ keystone_service_adminurl }}"
-    login_user: "{{ keystone_admin_user_name }}"
-    login_password: "{{ keystone_auth_admin_password }}"
-    login_project_name: "{{ keystone_admin_tenant_name }}"
-    description: "{{ gnocchi_service_project_description }}"
-    insecure: "{{ keystone_service_adminuri_insecure }}"
-  register: add_project
-  until: add_project|success
-  retries: 5
-  delay: 10
-  no_log: True
+# We set the python interpreter to the ansible runtime venv if
+# the delegation is to localhost so that we get access to the
+# appropriate python libraries in that venv. If the delegation
+# is to another host, we assume that it is accessible by the
+# system python instead.
+- name: Setup the service
+  delegate_to: "{{ gnocchi_service_setup_host }}"
+  vars:
+    ansible_python_interpreter: >-
+      {{ (gnocchi_service_setup_host == 'localhost') | ternary(ansible_playbook_python, ansible_python['executable']) }}
+  block:
+    - name: Add service project
+      os_project:
+        cloud: default
+        state: present
+        name: "{{ gnocchi_service_project_name }}"
+        domain_id: "{{ gnocchi_service_project_domain_id }}"
+        endpoint_type: admin
+        verify: "{{ not keystone_service_adminuri_insecure }}"
+      register: add_service
+      until: add_service is success
+      retries: 5
+      delay: 10
 
-# Create an admin user
-- name: Ensure Gnocchi user
-  keystone:
-    command: "ensure_user"
-    endpoint: "{{ keystone_service_adminurl }}"
-    login_user: "{{ keystone_admin_user_name }}"
-    login_password: "{{ keystone_auth_admin_password }}"
-    login_project_name: "{{ keystone_admin_tenant_name }}"
-    user_name: "{{ gnocchi_service_user_name }}"
-    tenant_name: "{{ gnocchi_service_project_name }}"
-    password: "{{ gnocchi_service_password }}"
-    insecure: "{{ keystone_service_adminuri_insecure }}"
-  register: add_user
-  until: add_user|success
-  retries: 5
-  delay: 10
-  no_log: True
+    - name: Add service user
+      os_user:
+        cloud: default
+        state: present
+        name: "{{ gnocchi_service_user_name }}"
+        password: "{{ gnocchi_service_password }}"
+        domain: default
+        default_project: "{{ gnocchi_service_project_name }}"
+        endpoint_type: admin
+        verify: "{{ not keystone_service_adminuri_insecure }}"
+      register: add_service
+      until: add_service is success
+      retries: 5
+      delay: 10
+      no_log: True
 
-# Add a role to the user
-- name: Ensure Gnocchi user maps to admin role
-  keystone:
-    command: "ensure_user_role"
-    endpoint: "{{ keystone_service_adminurl }}"
-    login_user: "{{ keystone_admin_user_name }}"
-    login_password: "{{ keystone_auth_admin_password }}"
-    login_project_name: "{{ keystone_admin_tenant_name }}"
-    user_name: "{{ gnocchi_service_user_name }}"
-    tenant_name: "{{ gnocchi_service_project_name }}"
-    role_name: "{{ gnocchi_role_name }}"
-    insecure: "{{ keystone_service_adminuri_insecure }}"
-  register: add_admin_role
-  until: add_admin_role|success
-  retries: 5
-  delay: 10
-  no_log: True
+    - name: Add service user to admin role
+      os_user_role:
+        cloud: default
+        state: present
+        user: "{{ gnocchi_service_user_name }}"
+        role: "{{ gnocchi_role_name }}"
+        project: "{{ gnocchi_service_project_name }}"
+        endpoint_type: admin
+        verify: "{{ not keystone_service_adminuri_insecure }}"
+      register: add_service
+      until: add_service is success
+      retries: 5
+      delay: 10

tasks/gnocchi_install.yml

@@ -33,19 +33,6 @@
       {% endfor %}
   when: gnocchi_developer_mode | bool
 
-- name: Install required pip packages
-  pip:
-    name: "{{ gnocchi_requires_pip_packages }}"
-    state: "{{ gnocchi_pip_package_state }}"
-    extra_args: >-
-      {{ gnocchi_developer_mode | ternary(pip_install_developer_constraints | default('--constraint /opt/developer-pip-constraints.txt'), '') }}
-      {{ (pip_install_upper_constraints is defined) | ternary('--constraint ' + pip_install_upper_constraints | default(''),'') }}
-      {{ pip_install_options | default('') }}
-  register: install_packages
-  until: install_packages|success
-  retries: 5
-  delay: 2
-
 - name: Retrieve checksum for venv download
   uri:
     url: "{{ gnocchi_venv_download_url | replace('tgz', 'checksum') }}"

tasks/gnocchi_service_setup.yml

@@ -13,45 +13,49 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Create a service
-- name: Ensure Gnocchi service
-  keystone:
-    command: "ensure_service"
-    endpoint: "{{ keystone_service_adminurl }}"
-    login_user: "{{ keystone_admin_user_name }}"
-    login_password: "{{ keystone_auth_admin_password }}"
-    login_project_name: "{{ keystone_admin_tenant_name }}"
-    service_name: "{{ gnocchi_service_name }}"
-    service_type: "{{ gnocchi_service_type }}"
-    description: "{{ gnocchi_service_description }}"
-    insecure: "{{ keystone_service_adminuri_insecure }}"
-  register: add_service
-  until: add_service|success
-  retries: 5
-  delay: 2
-  no_log: True
+# We set the python interpreter to the ansible runtime venv if
+# the delegation is to localhost so that we get access to the
+# appropriate python libraries in that venv. If the delegation
+# is to another host, we assume that it is accessible by the
+# system python instead.
+- name: Setup the service
+  delegate_to: "{{ gnocchi_service_setup_host }}"
+  vars:
+    ansible_python_interpreter: >-
+      {{ (gnocchi_service_setup_host == 'localhost') | ternary(ansible_playbook_python, ansible_python['executable']) }}
+  block:
+    - name: Add service to the keystone service catalog
+      os_keystone_service:
+        cloud: default
+        state: present
+        name: "{{ gnocchi_service_name }}"
+        service_type: "{{ gnocchi_service_type }}"
+        description: "{{ gnocchi_service_description }}"
+        endpoint_type: admin
+        verify: "{{ not keystone_service_adminuri_insecure }}"
+      register: add_service
+      until: add_service is success
+      retries: 5
+      delay: 10
 
-# Create an endpoint
-- name: Ensure Gnocchi endpoint
-  keystone:
-    command: "ensure_endpoint"
-    endpoint: "{{ keystone_service_adminurl }}"
-    login_user: "{{ keystone_admin_user_name }}"
-    login_password: "{{ keystone_auth_admin_password }}"
-    login_project_name: "{{ keystone_admin_tenant_name }}"
-    region_name: "{{ gnocchi_service_region }}"
-    service_name: "{{ gnocchi_service_name }}"
-    service_type: "{{ gnocchi_service_type }}"
-    insecure: "{{ keystone_service_adminuri_insecure }}"
-    endpoint_list:
-      - url: "{{ gnocchi_service_publicurl }}"
-        interface: "public"
-      - url: "{{ gnocchi_service_internalurl }}"
-        interface: "internal"
-      - url: "{{ gnocchi_service_adminurl }}"
-        interface: "admin"
-  register: add_endpoint
-  until: add_endpoint|success
-  retries: 5
-  delay: 10
-  no_log: True
+    - name: Add endpoints to keystone endpoint catalog
+      os_keystone_endpoint:
+        cloud: default
+        state: present
+        service: "{{ gnocchi_service_name }}"
+        endpoint_interface: "{{ item.interface }}"
+        url: "{{ item.url }}"
+        region: "{{ gnocchi_service_region }}"
+        endpoint_type: admin
+        verify: "{{ not keystone_service_adminuri_insecure }}"
+      register: add_service
+      until: add_service is success
+      retries: 5
+      delay: 10
+      with_items:
+        - interface: "public"
+          url: "{{ gnocchi_service_publicurl }}"
+        - interface: "internal"
+          url: "{{ gnocchi_service_internalurl }}"
+        - interface: "admin"
+          url: "{{ gnocchi_service_adminurl }}"

tests/host_vars/localhost.yml

@@ -15,5 +15,3 @@
 
 bridges:
   - "br-mgmt"
-
-ansible_python_interpreter: "/usr/bin/python2"

tests/test-gnocchi-functional.yml

@@ -16,47 +16,44 @@
 # Very basic testing using examples from http://gnocchi.xyz/rest.html
 
 - name: Playbook for functional testing of gnocchi
-  hosts: gnocchi_all
-  user: root
+  hosts: localhost
+  connection: local
   gather_facts: false
   vars:
-      gnocchi_api: "http://localhost:{{ gnocchi_service_port }}"
+    ansible_python_interpreter: "{{ ansible_playbook_python }}"
+  vars_files:
+    - common/test-vars.yml
   tasks:
-    - name: Install openstackclient
-      pip:
-        name: "python-openstackclient"
-        extra_args: >-
-          {{ gnocchi_developer_mode | ternary(pip_install_developer_constraints | default('--constraint /opt/developer-pip-constraints.txt'), '') }}
-          {{ pip_install_options | default('') }}
-
     - name: Check the gnocchi-api
       uri:
-        url: "{{ gnocchi_api }}"
+        url: "{{ gnocchi_service_internaluri }}"
         status_code: 200,300
 
     - name: Validate that auth is required
       uri:
-        url: "{{ gnocchi_api }}/v1/status"
+        url: "{{ gnocchi_service_internaluri }}/v1/status"
         status_code: 401
 
-    - name: Get auth token
-      shell: >
-        . /root/openrc && openstack token issue --format yaml | awk '/^id\:/ {print $2}'
-      register: get_keystone_token
-      changed_when: false
-
-    - name: set token
-      set_fact:
-        keystone_token: "{{ get_keystone_token.stdout }}"
+    - name: Authenticate to the cloud and retrieve the service catalog
+      os_auth:
+        cloud: "default"
+        region_name: "{{ keystone_service_region }}"
+      # TODO(odyssey4me):
+      # Restore this once debugging is complete.
+      #no_log: true
+      register: _auth
+      until: (_auth | success) and (auth_token is defined)
+      retries: 5
+      delay: 10
 
     - name: Create a metric
       uri:
-        url: "{{ gnocchi_api }}/v1/metric"
+        url: "{{ gnocchi_service_internaluri }}/v1/metric"
         method: POST
         body: '{ "archive_policy_name": "high" }'
         headers:
           Content-Type: "application/json"
-          X-Auth-Token: "{{ keystone_token }}"
+          X-Auth-Token: "{{ auth_token }}"
         return_content: True
         status_code: 201
       register: metric_create
@@ -66,22 +63,22 @@
 
     - name: Add measures
       uri:
-        url: "{{ gnocchi_api }}/v1/metric/{{ metric_create.json.id }}/measures"
+        url: "{{ gnocchi_service_internaluri }}/v1/metric/{{ metric_create.json.id }}/measures"
         method: POST
         body: '[ { "timestamp": "2014-10-06T14:33:57", "value": 43.1 }, { "timestamp": "2014-10-06T14:34:12", "value": 12 }, { "timestamp": "2014-10-06T14:34:20", "value": 2 } ]'
         headers:
           Content-Type: "application/json"
-          X-Auth-Token: "{{ keystone_token }}"
+          X-Auth-Token: "{{ auth_token }}"
         return_content: True
         status_code: 202
 
     - name: Retrieve the measures
       uri:
-        url: "{{ gnocchi_api }}/v1/metric/{{ metric_create.json.id }}/measures?refresh=true"
+        url: "{{ gnocchi_service_internaluri }}/v1/metric/{{ metric_create.json.id }}/measures?refresh=true"
         method: GET
         headers:
           Content-Type: "application/json"
-          X-Auth-Token: "{{ keystone_token }}"
+          X-Auth-Token: "{{ auth_token }}"
         return_content: True
         status_code: 200
       register: measures_retrieval
@@ -96,11 +93,11 @@
 
     - name: Retrieve the archive policies
       uri:
-        url: "{{ gnocchi_api }}/v1/archive_policy"
+        url: "{{ gnocchi_service_internaluri }}/v1/archive_policy"
         method: GET
         headers:
           Content-Type: "application/json"
-          X-Auth-Token: "{{ keystone_token }}"
+          X-Auth-Token: "{{ auth_token }}"
         return_content: True
         status_code: 200
       register: policies_retrieval

tests/test-install-gnocchi.yml

@@ -17,11 +17,11 @@
   hosts: gnocchi_all
   user: root
   gather_facts: true
+  vars_files:
+    - common/test-vars.yml
   pre_tasks:
     - include: common/create-grant-db.yml
       db_password: "{{ gnocchi_container_mysql_password }}"
       db_name: "gnocchi"
   roles:
     - role: "os_gnocchi"
-  vars_files:
-    - common/test-vars.yml