Added support for Hiera to multiple backends

Templating from hiera to config files
Modular main.ym

defaults/main.yml

@@ -4,6 +4,7 @@ config_file: "~/.kube/config"
 action: provision
 namespace: openstack
 service_account: openstack
+privileged_service_account: openstack-priv
 database_password: weakpassword
 cinder_db_password: cinderpass
 
@@ -16,3 +17,4 @@ cinder_config:
   database:
     connection: "mysql+pymysql://root:weakpassword@mariadb:3306/cinder"
 
+enabled_backends: []

tasks/backends/iscsi.yml

@@ -0,0 +1,209 @@
+- name: Upload config files
+  template:
+    src: "{{item}}"
+    dest: /tmp/{{item}}
+    backup: yes
+    mode: 0644
+  with_items:
+    - lvm.conf
+
+- name: Read configs into memory
+  slurp:
+    src: "/tmp/lvm.conf"
+  register: "lvm_conf"
+
+- name: Create iscsid configmap
+  ignore_errors: yes
+  k8s_v1_config_map:
+    name: iscsid
+    namespace: '{{ namespace }}'
+    state: present
+    verify_ssl: false
+    labels:
+      app: iscsid
+      service: iscsid
+    data:
+      config.json: |
+        {
+            "command": "/usr/sbin/iscsid -f",
+            "config_files": [
+                {
+                    "source": "/var/lib/kolla/config_files/initiatorname.iscsi",
+                    "dest": "/etc/iscsi/initiatorname.iscsi",
+                    "owner": "root",
+                    "perm": "0600"
+                }
+            ]
+        }
+
+      initiatorname.iscsi: |
+        InitiatorName=iqn.1994-05.com.redhat:da4a2a2d0d1
+
+- name: Create cinder-volume-iscsi configmaps
+  ignore_errors: yes
+  k8s_v1_config_map:
+    name: cinder-volume-iscsi
+    namespace: "{{ namespace }}"
+    state: present
+    debug: yes
+    labels:
+      service: cinder-volume-iscsi
+    data:
+      config.json: |
+        {
+            "command": "/usr/bin/cinder-volume --config-file /usr/share/cinder/cinder-dist.conf --config-file /etc/cinder/cinder.conf",
+            "config_files": [
+                {
+                    "source": "/var/lib/kolla/config_files/cinder.conf",
+                    "dest": "/etc/cinder/cinder.conf",
+                    "owner": "cinder",
+                    "perm": "0600"
+                },
+                {
+                    "source": "/var/lib/kolla/config_files/lvm.conf",
+                    "dest": "/etc/lvm/lvm.conf",
+                    "owner": "root",
+                    "perm": "0644"
+                }
+            ],
+            "permissions": [
+                {
+                    "path": "/var/lib/cinder",
+                    "owner": "cinder:cinder",
+                    "recurse": true
+                },
+                {
+                    "path": "/var/log/kolla/cinder",
+                    "owner": "cinder:cinder",
+                    "recurse": true
+                }
+            ]
+        }
+
+      cinder.conf: |
+        {{cinder_conf['content'] | b64decode}}
+
+      lvm.conf: |
+        {{lvm_conf['content'] | b64decode}}
+
+
+- name: Create Cinder Volume Deployment
+  k8s_v1beta1_deployment:
+    name: cinder-volume-iscsi
+    namespace: "{{ namespace }}"
+    service_account_name: '{{ privileged_service_account }}'
+    labels:
+      app: cinder-volume-iscsi
+      service: cinder-volume-iscsi
+    replicas: 1
+    spec_template_metadata_labels:
+      app: cinder-volume-iscsi
+      service: cinder-volume-iscsi
+    containers:
+    - name: cinder-volume
+      securityContext:
+        privileged: true
+      image: tripleoupstream/centos-binary-cinder-volume:latest
+      image_pull_policy: IfNotPresent
+      volumeMounts:
+        - name: kolla-config
+          mountPath: /var/lib/kolla/config_files/
+        - mountPath: /etc/localtime
+          name: host-etc-localtime
+          readOnly: true
+        - mountPath: /run
+          name: run
+        - mountPath: /dev
+          name: dev
+        - mountPath: /sys/fs/cgroup
+          name: sys-fs-cgroup
+        - mountPath: /lib/modules
+          name: lib-modules
+      env:
+        - name: TZ
+          value: UTC
+        - name: KOLLA_CONFIG_STRATEGY
+          value: COPY_ALWAYS
+        - name: KOLLA_KUBERNETES
+          value: ""
+    volumes:
+      - name: host-etc-localtime
+        hostPath:
+          path: /etc/localtime
+      - name: lib-modules
+        hostPath:
+          path: /lib/modules
+      - name: run
+        hostPath:
+          path: /run
+      - name: dev
+        hostPath:
+          path: /dev
+      - name: sys-fs-cgroup
+        hostPath:
+          path: /sys/fs/cgroup
+      - name: kolla-config
+        config_map:
+          name: cinder-volume-iscsi
+
+- name: Create Iscsid Deployment
+  k8s_v1beta1_deployment:
+    name: iscsid
+    namespace: '{{ namespace }}'
+    service_account_name: '{{ privileged_service_account }}'
+    node_selector:
+      controller: "true"
+    labels:
+      app: iscsid
+      service: iscsid
+    replicas: 1
+    spec_template_metadata_labels:
+      app: iscsid
+      service: iscsid
+    containers:
+    - name: iscsid
+      securityContext:
+        privileged: true
+      image: tripleoupstream/centos-binary-iscsid:latest
+      image_pull_policy: IfNotPresent
+      volumeMounts:
+        - mountPath: /etc/localtime
+          name: host-etc-localtime
+          readOnly: true
+        - mountPath: /run
+          name: run
+        - mountPath: /dev
+          name: dev
+        - mountPath: /sys/fs/cgroup
+          name: sys-fs-cgroup
+        - mountPath: /lib/modules
+          name: lib-modules
+        - name: kolla-config
+          mountPath: /var/lib/kolla/config_files/
+      env:
+        - name: TZ
+          value: UTC
+        - name: KOLLA_CONFIG_STRATEGY
+          value: COPY_ALWAYS
+        - name: KOLLA_KUBERNETES
+          value: ""
+    volumes:
+      - name: host-etc-localtime
+        hostPath:
+          path: /etc/localtime
+      - name: lib-modules
+        hostPath:
+          path: /lib/modules
+      - name: run
+        hostPath:
+          path: /run
+      - name: dev
+        hostPath:
+          path: /dev
+      - name: sys-fs-cgroup
+        hostPath:
+          path: /sys/fs/cgroup
+      - name: kolla-config
+        config_map:
+          name: iscsid
+

tasks/backends/rbd.yml

@@ -0,0 +1,87 @@
+- name: Create cinder-volume-rbd configmaps
+  ignore_errors: yes
+  k8s_v1_config_map:
+    name: cinder-volume-rbd
+    namespace: "{{ namespace }}"
+    state: present
+    debug: yes
+    labels:
+      service: cinder-volume-rbd
+    data:
+      config.json: |
+        {
+            "command": "/usr/bin/cinder-volume --config-file /usr/share/cinder/cinder-dist.conf --config-file /etc/cinder/cinder.conf",
+            "config_files": [
+                {
+                    "source": "/var/lib/kolla/config_files/cinder.conf",
+                    "dest": "/etc/cinder/cinder.conf",
+                    "owner": "cinder",
+                    "perm": "0600"
+                },
+                {
+                    "source": "/var/lib/kolla/config_files/ceph.conf",
+                    "dest": "/etc/ceph/ceph.conf",
+                    "owner": "root",
+                    "perm": "0644"
+                },
+                {
+                    "source": "/var/lib/kolla/config_files/ceph.client.wshed.keyring",
+                    "dest": "/etc/ceph/ceph.client.wshed.keyring",
+                    "owner": "root",
+                    "perm": "0644"
+                }
+            ],
+            "permissions": [
+                {
+                    "path": "/var/lib/cinder",
+                    "owner": "cinder:cinder",
+                    "recurse": true
+                },
+                {
+                    "path": "/var/log/kolla/cinder",
+                    "owner": "cinder:cinder",
+                    "recurse": true
+                }
+            ]
+        }
+
+      cinder.conf: |
+        {{cinder_conf['content'] | b64decode}}
+
+      ceph.conf: |
+        {{ceph_conf['content'] | b64decode}}
+
+      ceph.client.wshed.keyring: |
+        {{ceph_client_keyring['content'] | b64decode}}
+
+- name: Create Cinder Volume Deployment
+  k8s_v1beta1_deployment:
+    name: cinder-volume-rbd
+    namespace: "{{ namespace }}"
+    service_account_name: "{{ service_account }}"
+    labels:
+      app: cinder-volume-rbd
+      service: cinder-volume-rbd
+    replicas: 1
+    spec_template_metadata_labels:
+      app: cinder-volume-rbd
+      service: cinder-volume-rbd
+    containers:
+    - name: cinder-volume
+      image: tripleoupstream/centos-binary-cinder-volume:latest
+      image_pull_policy: IfNotPresent
+      volumeMounts:
+        - name: kolla-config
+          mountPath: /var/lib/kolla/config_files/
+      env:
+        - name: TZ
+          value: UTC
+        - name: KOLLA_CONFIG_STRATEGY
+          value: COPY_ALWAYS
+        - name: KOLLA_KUBERNETES
+          value: ""
+    volumes:
+      - name: kolla-config
+        config_map:
+          name: cinder-volume-rbd
+

tasks/backends/xtremio.yml

@@ -0,0 +1,69 @@
+- name: Create cinder-volume-xtremio configmaps
+  ignore_errors: yes
+  k8s_v1_config_map:
+    name: cinder-volume-xtremio
+    namespace: "{{ namespace }}"
+    state: present
+    debug: yes
+    labels:
+      service: cinder-volume-xtremio
+    data:
+      config.json: |
+        {
+            "command": "/usr/bin/cinder-volume --config-file /usr/share/cinder/cinder-dist.conf --config-file /etc/cinder/cinder.conf",
+            "config_files": [
+                {
+                    "source": "/var/lib/kolla/config_files/cinder.conf",
+                    "dest": "/etc/cinder/cinder.conf",
+                    "owner": "cinder",
+                    "perm": "0600"
+                }
+            ],
+            "permissions": [
+                {
+                    "path": "/var/lib/cinder",
+                    "owner": "cinder:cinder",
+                    "recurse": true
+                },
+                {
+                    "path": "/var/log/kolla/cinder",
+                    "owner": "cinder:cinder",
+                    "recurse": true
+                }
+            ]
+        }
+
+      cinder.conf: |
+        {{cinder_conf['content'] | b64decode}}
+
+- name: Create Cinder Volume Deployment
+  k8s_v1beta1_deployment:
+    name: cinder-volume-xtremio
+    namespace: "{{ namespace }}"
+    service_account_name: "{{ service_account }}"
+    labels:
+      app: cinder-volume-xtremio
+      service: cinder-volume-xtremio
+    replicas: 1
+    spec_template_metadata_labels:
+      app: cinder-volume-xtremio
+      service: cinder-volume-xtremio
+    containers:
+    - name: cinder-volume
+      image: tripleoupstream/centos-binary-cinder-volume:latest
+      image_pull_policy: IfNotPresent
+      volumeMounts:
+        - name: kolla-config
+          mountPath: /var/lib/kolla/config_files/
+      env:
+        - name: TZ
+          value: UTC
+        - name: KOLLA_CONFIG_STRATEGY
+          value: COPY_ALWAYS
+        - name: KOLLA_KUBERNETES
+          value: ""
+    volumes:
+      - name: kolla-config
+        config_map:
+          name: cinder-volume-xtremio
+

tasks/cinder-api.yml

@@ -0,0 +1,116 @@
+# Upload and create pod specific configmap
+- name: Upload config files
+  template:
+    src: "{{item}}"
+    dest: /tmp/{{item}}
+    backup: yes
+    mode: 0644
+  with_items:
+    - cinder-wsgi.conf
+
+- name: Read configs into memory
+  slurp:
+    src: "/tmp/cinder-wsgi.conf"
+  register: "cinder_wsgi_conf"
+
+- name: Create cinder-api configmaps
+  ignore_errors: yes
+  k8s_v1_config_map:
+    name: cinder-api
+    namespace: "{{ namespace }}"
+    state: present
+    debug: yes
+    labels:
+      service: cinder
+    data:
+      config.json: |
+        {
+            "command": "httpd -DFOREGROUND",
+            "config_files": [
+                {
+                    "source": "/var/lib/kolla/config_files/cinder.conf",
+                    "dest": "/etc/cinder/cinder.conf",
+                    "owner": "cinder",
+                    "perm": "0600"
+                },
+                {
+                    "source": "/var/lib/kolla/config_files/cinder-wsgi.conf",
+                    "dest": "/etc/httpd/conf.d/cinder-wsgi.conf",
+                    "owner": "cinder",
+                    "perm": "0600"
+                }
+            ],
+            "permissions": [
+                {
+                    "path": "/var/lib/cinder",
+                    "owner": "cinder:cinder",
+                    "recurse": true
+                },
+                {
+                    "path": "/var/log/kolla/cinder",
+                    "owner": "cinder:cinder",
+                    "recurse": true
+                }
+            ]
+        }
+
+      cinder.conf: |
+        {{cinder_conf['content'] | b64decode}}
+
+      cinder-wsgi.conf: |
+        {{cinder_wsgi_conf['content'] | b64decode}}
+
+- name: create cinder-api service
+  k8s_v1_service:
+    name: cinder
+    host: "{{coe_host}}"
+    context: "{{kube_context}}"
+    kubeconfig: "{{config_file}}"
+    namespace: "{{ namespace }}"
+    labels:
+      app: cinder-api
+      service: cinder-api
+    selector:
+      app: cinder-api
+      service: cinder-api
+    ports:
+      - name: cinder-api
+        port: 8776
+        target_port: 8776
+
+- name: Create Cinder-api Deployment
+  k8s_v1beta1_deployment:
+    name: cinder-api
+    host: "{{coe_host}}"
+    context: "{{kube_context}}"
+    kubeconfig: "{{config_file}}"
+    namespace: "{{ namespace }}"
+    service_account_name: "{{ service_account }}"
+    labels:
+      app: cinder-api
+      service: cinder-api
+    replicas: 1
+    spec_template_metadata_labels:
+      app: cinder-api
+      service: cinder-api
+    containers:
+    - name: cinder-api
+      image: tripleoupstream/centos-binary-cinder-api:latest
+      ports:
+        - container_port: 8776
+          protocol: TCP
+      env:
+        - name: TZ
+          value: UTC
+        - name: KOLLA_CONFIG_STRATEGY
+          value: COPY_ALWAYS
+        - name: KOLLA_KUBERNETES
+          value: ""
+      volume_mounts:
+        - name: kolla-config
+          mountPath: /var/lib/kolla/config_files/
+    volumes:
+      - name: kolla-config
+        config_map:
+          name: cinder-api
+

tasks/cinder-scheduler.yml

@@ -0,0 +1,68 @@
+- name: Create cinder-scheduler configmaps
+  ignore_errors: yes
+  k8s_v1_config_map:
+    name: cinder-scheduler
+    namespace: "{{ namespace }}"
+    state: present
+    debug: yes
+    labels:
+      service: cinder-scheduler
+    data:
+      config.json: |
+        {
+            "command": "cinder-scheduler --config-file /var/lib/kolla/config_files/cinder.conf",
+            "config_files": [
+                {
+                    "source": "/var/lib/kolla/config_files/cinder.conf",
+                    "dest": "/etc/cinder/cinder.conf",
+                    "owner": "cinder",
+                    "perm": "0600"
+                }
+            ],
+            "permissions": [
+                {
+                    "path": "/var/lib/cinder",
+                    "owner": "cinder:cinder",
+                    "recurse": true
+                },
+                {
+                    "path": "/var/log/kolla/cinder",
+                    "owner": "cinder:cinder",
+                    "recurse": true
+                }
+            ]
+        }
+
+      cinder.conf: |
+        {{cinder_conf['content'] | b64decode}}
+
+- name: create cinder-scheduler deployment
+  k8s_v1beta1_deployment:
+    name: cinder-scheduler
+    namespace: "{{ namespace }}"
+    service_account_name: "{{ service_account }}"
+    labels:
+      app: cinder-scheduler
+      service: cinder-scheduler
+    replicas: 1
+    spec_template_metadata_labels:
+      app: cinder-scheduler
+      service: cinder-scheduler
+    containers:
+    - name: cinder-scheduler
+      image: tripleoupstream/centos-binary-cinder-scheduler:latest
+      env:
+        - name: TZ
+          value: UTC
+        - name: KOLLA_CONFIG_STRATEGY
+          value: COPY_ALWAYS
+        - name: KOLLA_KUBERNETES
+          value: ""
+      volume_mounts:
+        - name: kolla-config
+          mountPath: /var/lib/kolla/config_files/
+    volumes:
+      - name: kolla-config
+        config_map:
+          name: cinder-scheduler
+

tasks/cinder-volume.yml

@@ -0,0 +1,9 @@
+# Ensure that there is a 1:1 mapping of cinder-volume
+# backend configmap to its corresponding cinder-volume pod
+
+- include: backends/iscsi.yml
+  when: result.conf_dict['cinder_enable_iscsi_backend']
+- include: backends/rbd.yml
+  when: result.conf_dict['cinder_enable_rbd_backend']
+- include: backends/xtremio.yml
+  when: result.conf_dict['dellemc_xtremio_iscsi_backend']

tasks/enabled_backends.yml

@@ -0,0 +1,16 @@
+# Gather list of enabled backends here
+- name: Append backend to enable_backends list
+  set_fact:
+    enabled_backends: "{{ enabled_backends }} + [ 'tripleo_ceph' ]"
+  when: result.conf_dict['cinder_enable_rbd_backend']
+
+- name: Append backend to enable_backends list
+  set_fact:
+    enabled_backends: "{{ enabled_backends }} + [ 'tripleo_iscsi' ]"
+  when: result.conf_dict['cinder_enable_iscsi_backend']
+
+- name: Append backend to enable_backends list
+  set_fact:
+    enabled_backends: "{{ enabled_backends }} + [ 'xtremio' ]"
+  when: result.conf_dict['dellemc_xtremio_iscsi_backend']
+

tasks/hiera.yml

@@ -2,7 +2,22 @@
   include_role:
     name: 'ansible-role-k8s-tripleo'
   vars:
-    hieradata: '{{hiera_data}}'
+    hieradata: 
+      tripleo::profile::base::cinder::volume::dellemc_xtremio_iscsi_backend: true
+      tripleo::profile::base::cinder::volume::cinder_enable_iscsi_backend: true
+      tripleo::profile::base::cinder::volume::cinder_enable_rbd_backend: false
+      tripleo::profile::base::cinder::volume::cinder_enable_nfs_backend: false
+      tripleo::profile::base::cinder::volume::iscsi::cinder_lvm_loop_device_size: 10280
+      tripleo::profile::base::cinder::volume::iscsi::cinder_iscsi_helper: lioadm
+      tripleo::profile::base::cinder::volume::iscsi::cinder_iscsi_address: iscsid
+      tripleo::profile::base::cinder::volume::rbd::cinder_rbd_pool_name: volumes
+      tripleo::profile::base::cinder::volume::rbd::cinder_rbd_user_name: openstack
+      ceph::profile::params::fsid: 1611b996-d2d7-11e6-b1c2-5254004237d4
+      cinder::rabbit_userid: guest
+      cinder::rabbit_port: 5672
+      cinder::rabbit_password: rabbitpass
+
+
     hieradata_file: '{{hiera_data_file}}'
     schema:
       cinder::api::bind_port: bind_port
@@ -17,5 +32,33 @@
       cinder::database_connection: database.connection
       cinder::db::database_max_retries: database.max_retries
       cinder::db::database_db_max_retries: database.db_max_retries
+
+      tripleo::profile::base::cinder::volume::cinder_enable_iscsi_backend: cinder_enable_iscsi_backend
+      tripleo::profile::base::cinder::volume::cinder_enable_nfs_backend: cinder_enable_nfs_backend
+      tripleo::profile::base::cinder::volume::cinder_enable_rbd_backend: cinder_enable_rbd_backend
+      tripleo::profile::base::cinder::volume::nfs::cinder_nfs_mount_options: cinder_nfs_mount_options
+      tripleo::profile::base::cinder::volume::nfs::cinder_nfs_servers: cinder_nfs_servers
+      tripleo::profile::base::cinder::volume::nfs::cinder_nas_secure_file_operations: cinder_nas_secure_file_operations
+      tripleo::profile::base::cinder::volume::nfs::cinder_nas_secure_file_permissions: cinder_nas_secure_file_permissions
+      tripleo::profile::base::cinder::volume::iscsi::cinder_lvm_loop_device_size: cinder_lvm_loop_device_size
+      tripleo::profile::base::cinder::volume::iscsi::cinder_iscsi_helper: cinder_iscsi_helper
+      tripleo::profile::base::cinder::volume::iscsi::cinder_iscsi_protocol: cinder_iscsi_protocol
+      tripleo::profile::base::cinder::volume::iscsi::cinder_iscsi_address: cinder_iscsi_address
+      tripleo::profile::base::cinder::volume::rbd::cinder_rbd_pool_name: cinder_rbd_pool_name
+      tripleo::profile::base::cinder::volume::rbd::cinder_rbd_user_name: cinder_rbd_user_name
+
+      tripleo::profile::base::cinder::volume::dellemc_xtremio_iscsi_backend: dellemc_xtremio_iscsi_backend
+      cinder::backend::dellemc_xtremio_iscsi::max_over_subscription_ratio: xtremio.max_over_subscription_ratio
+      cinder::backend::dellemc_xtremio_iscsi::use_multipath_for_image_xfer: xtremio.use_multipath_for_image_xfer
+      cinder::backend::dellemc_xtremio_iscsi::volume_backend_name: xtremio.volume_backend_name
+      cinder::backend::dellemc_xtremio_iscsi::scan_ip: xtremio.scan_ip
+      cinder::backend::dellemc_xtremio_iscsi::xtremio_cluster_name: xtremio.xtremio_cluster_name
+      cinder::backend::dellemc_xtremio_iscsi::san_login: xtremio.san_login
+      cinder::backend::dellemc_xtremio_iscsi::san_password: xtremio.san_password
+      cinder::backend::dellemc_xtremio_iscsi::image_volume_cache_enabled: xtremio.image_volume_cache_enabled
+
+      ceph::profile::params::fsid: ceph_cluster_fsid
+
     default: '{{cinder_config}}'
     fact_variable: 'cinder_config'
+

tasks/init-jobs.yml

@@ -0,0 +1,133 @@
+# List of init jobs for the pod go here
+- name: Create cinder-init-job configmaps
+  ignore_errors: yes
+  k8s_v1_config_map:
+    name: cinder-init-job
+    namespace: "{{ namespace }}"
+    state: present
+    debug: yes
+    data:
+      config.json: |
+        {
+            "command": "cinder-scheduler --config-file /etc/cinder/cinder.conf",
+            "config_files": [
+                {
+                    "source": "/var/lib/kolla/config_files/cinder.conf",
+                    "dest": "/etc/cinder/cinder.conf",
+                    "owner": "cinder",
+                    "perm": "0600"
+                }
+            ],
+            "permissions": [
+                {
+                    "path": "/var/lib/cinder",
+                    "owner": "cinder:cinder",
+                    "recurse": true
+                },
+                {
+                    "path": "/var/log/kolla/cinder",
+                    "owner": "cinder:cinder",
+                    "recurse": true
+                }
+            ]
+        }
+
+      cinder.conf: |
+        {{cinder_conf['content'] | b64decode}}
+
+- name: Create Cinder Database
+  k8s_v1_job:
+    name: cinder-create-db
+    namespace: "{{ namespace }}"
+    service_account_name: "{{ service_account }}"
+    restart_policy: OnFailure
+    containers:
+    - name: cinder-create-db
+      image: tripleoupstream/centos-binary-kolla-toolbox:latest
+      image_pull_policy: IfNotPresent
+      command: ["sh", "-c"]
+      args:
+        - ansible localhost -m mysql_db -a
+          "login_host='mariadb'
+           login_port='3306'
+           login_user='root'
+           login_password='$DATABASE_PASSWORD'
+           name='cinder'"
+      env:
+        - name: DATABASE_PASSWORD
+          value: "{{ database_password }}"
+        - name: "TZ"
+          value: "UTC"
+  register: db_create
+
+- name: Create Cinder User in the database
+  k8s_v1_job:
+    name: cinder-create-user
+    namespace: "{{ namespace }}"
+    service_account_name: "{{ service_account }}"
+    restart_policy: OnFailure
+    containers:
+    - name: cinder-create-db
+      image: tripleoupstream/centos-binary-kolla-toolbox:latest
+      image_pull_policy: IfNotPresent
+      command: ["sh", "-c"]
+      args:
+        - ansible localhost -m mysql_user -a
+          "login_host='mariadb'
+           login_port='3306'
+           login_user='root'
+           login_password='$DATABASE_PASSWORD'
+           name='cinder'
+           password='$CINDER_DATABASE_PASSWORD'
+           host='%'
+           priv='cinder.*:ALL'
+           append_privs='yes'"
+      env:
+        - name: CINDER_DATABASE_PASSWORD
+          value: "{{ cinder_db_password }}"
+        - name: DATABASE_PASSWORD
+          value: "{{database_password}}"
+        - name: "TZ"
+          value: "UTC"
+  when: db_create|succeeded
+
+# Cinder Scheduler won't start without cinder tables present
+- name: Create Cinder Tables
+  k8s_v1_job:
+    name: cinder-create-tables
+    namespace: "{{ namespace }}"
+    service_account_name: "{{ service_account }}"
+    restart_policy: OnFailure
+    containers:
+    - name: cinder-create-db
+      image: tripleoupstream/centos-binary-cinder-api:latest
+      image_pull_policy: IfNotPresent
+#      command: ["/usr/bin/cinder-manage", "db", "sync"]
+      env:
+        - name: KOLLA_KUBERNETES
+          value: ""
+        - name: KOLLA_BOOTSTRAP
+          value: ""
+        - name: KOLLA_CONFIG_STRATEGY
+          value: "COPY_ONCE"
+        - name: "TZ"
+          value: "UTC"
+      volume_mounts:
+        - name: kolla-config
+          mountPath: /var/lib/kolla/config_files/
+    volumes:
+      - name: kolla-config
+        config_map:
+          name: cinder-init-job
+  when: db_create|succeeded
+  register: cinder_bootstrap
+
+# Remove the init-job configmap.
+- name: Delete cinder-init-job configmaps
+  ignore_errors: yes
+  k8s_v1_config_map:
+    name: cinder-init-job
+    namespace: "{{namespace}}"
+    state: absent
+  when: cinder_bootstrap|succeeded
+

tasks/provision.yml

@@ -1,13 +1,6 @@
-- name: Upload config files
-  template:
-    src: "{{item}}"
-    dest: /tmp/{{item}}
-    backup: yes
-    mode: 0644
-  with_items:
-    - cinder-wsgi.conf
 
 - include: hiera.yml
+- include: enabled_backends.yml
 
 - name: Generate config files
   config_template:
@@ -21,357 +14,8 @@
     src: "/tmp/cinder.conf"
   register: "cinder_conf"
 
-- name: Read configs into memory
-  slurp:
-    src: "/tmp/cinder-wsgi.conf"
-  register: "cinder_wsgi_conf"
+- include: init-jobs.yml
+- include: cinder-api.yml
+- include: cinder-scheduler.yml
+- include: cinder-volume.yml
 
-- name: Create cinder configmaps
-  ignore_errors: yes
-  k8s_v1_config_map:
-    name: cinder-api
-    host: "{{coe_host}}"
-    context: "{{kube_context}}"
-    kubeconfig: "{{config_file}}"
-    namespace: "{{ namespace }}"
-    state: present
-    debug: yes
-    labels:
-      service: cinder
-    data:
-      config.json: |
-        {
-            "command": "httpd -DFOREGROUND",
-            "config_files": [
-                {
-                    "source": "/var/lib/kolla/config_files/cinder.conf",
-                    "dest": "/etc/cinder/cinder.conf",
-                    "owner": "cinder",
-                    "perm": "0600"
-                },
-                {
-                    "source": "/var/lib/kolla/config_files/cinder-wsgi.conf",
-                    "dest": "/etc/httpd/conf.d/cinder-wsgi.conf",
-                    "owner": "cinder",
-                    "perm": "0600"
-                }
-            ],
-            "permissions": [
-                {
-                    "path": "/var/lib/cinder",
-                    "owner": "cinder:cinder",
-                    "recurse": true
-                },
-                {
-                    "path": "/var/log/kolla/cinder",
-                    "owner": "cinder:cinder",
-                    "recurse": true
-                }
-            ]
-        }
-
-      cinder.conf: |
-        {{cinder_conf['content'] | b64decode}}
-
-      cinder-wsgi.conf: |
-        {{cinder_wsgi_conf['content'] | b64decode}}
-
-- name: Create cinder-scheduler configmaps
-  ignore_errors: yes
-  k8s_v1_config_map:
-    name: cinder-scheduler
-    host: "{{coe_host}}"
-    context: "{{kube_context}}"
-    kubeconfig: "{{config_file}}"
-    namespace: "{{ namespace }}"
-    state: present
-    debug: yes
-    labels:
-      service: cinder-scheduler
-    data:
-      config.json: |
-        {
-            "command": "cinder-scheduler --config-file /etc/cinder/cinder.conf",
-            "config_files": [
-                {
-                    "source": "/var/lib/kolla/config_files/cinder.conf",
-                    "dest": "/etc/cinder/cinder.conf",
-                    "owner": "cinder",
-                    "perm": "0600"
-                }
-            ],
-            "permissions": [
-                {
-                    "path": "/var/lib/cinder",
-                    "owner": "cinder:cinder",
-                    "recurse": true
-                },
-                {
-                    "path": "/var/log/kolla/cinder",
-                    "owner": "cinder:cinder",
-                    "recurse": true
-                }
-            ]
-        }
-
-      cinder.conf: |
-        {{cinder_conf['content'] | b64decode}}
-
-- name: Create cinder-volume configmaps
-  ignore_errors: yes
-  k8s_v1_config_map:
-    name: cinder-volume
-    host: "{{coe_host}}"
-    context: "{{kube_context}}"
-    kubeconfig: "{{config_file}}"
-    namespace: "{{ namespace }}"
-    state: present
-    debug: yes
-    labels:
-      service: cinder-volume
-    data:
-      config.json: |
-        {
-            "command": "/usr/bin/cinder-volume --config-file /usr/share/cinder/cinder-dist.conf --config-file /etc/cinder/cinder.conf",
-            "config_files": [
-                {
-                    "source": "/var/lib/kolla/config_files/cinder.conf",
-                    "dest": "/etc/cinder/cinder.conf",
-                    "owner": "cinder",
-                    "perm": "0600"
-                }
-            ],
-            "permissions": [
-                {
-                    "path": "/var/lib/cinder",
-                    "owner": "cinder:cinder",
-                    "recurse": true
-                },
-                {
-                    "path": "/var/log/kolla/cinder",
-                    "owner": "cinder:cinder",
-                    "recurse": true
-                }
-            ]
-        }
-
-      cinder.conf: |
-        {{cinder_conf['content'] | b64decode}}
-
-- name: Create Cinder Database
-  k8s_v1_job:
-    name: cinder-create-db
-    host: "{{coe_host}}"
-    context: "{{kube_context}}"
-    kubeconfig: "{{config_file}}"
-    namespace: "{{ namespace }}"
-    service_account_name: "{{ service_account }}"
-    restart_policy: OnFailure
-    containers:
-    - name: cinder-create-db
-      image: tripleoupstream/centos-binary-kolla-toolbox:latest
-      image_pull_policy: IfNotPresent
-      command: ["sh", "-c"]
-      args:
-        - ansible localhost -m mysql_db -a
-          "login_host='mariadb'
-           login_port='3306'
-           login_user='root'
-           login_password='$DATABASE_PASSWORD'
-           name='cinder'"
-      env:
-        - name: DATABASE_PASSWORD
-          value: "{{ database_password }}"
-        - name: "TZ"
-          value: "UTC"
-  register: result
-
-- name: Create Cinder User in the database
-  k8s_v1_job:
-    name: cinder-create-user
-    host: "{{coe_host}}"
-    context: "{{kube_context}}"
-    kubeconfig: "{{config_file}}"
-    namespace: "{{ namespace }}"
-    service_account_name: "{{ service_account }}"
-    restart_policy: OnFailure
-    containers:
-    - name: cinder-create-db
-      image: tripleoupstream/centos-binary-kolla-toolbox:latest
-      image_pull_policy: IfNotPresent
-      command: ["sh", "-c"]
-      args:
-        - ansible localhost -m mysql_user -a
-          "login_host='mariadb'
-           login_port='3306'
-           login_user='root'
-           login_password='$DATABASE_PASSWORD'
-           name='cinder'
-           password='$CINDER_DATABASE_PASSWORD'
-           host='%'
-           priv='cinder.*:ALL'
-           append_privs='yes'"
-      env:
-        - name: CINDER_DATABASE_PASSWORD
-          value: "{{ cinder_db_password }}"
-        - name: DATABASE_PASSWORD
-          value: "{{database_password}}"
-        - name: "TZ"
-          value: "UTC"
-  when: result|succeeded
-
-# Cinder Scheduler won't start without cinder tables present
-- name: Create Cinder Tables
-  k8s_v1_job:
-    name: cinder-create-tables
-    host: "{{coe_host}}"
-    context: "{{kube_context}}"
-    kubeconfig: "{{config_file}}"
-    namespace: "{{ namespace }}"
-    service_account_name: "{{ service_account }}"
-    restart_policy: OnFailure
-    containers:
-    - name: cinder-create-db
-      image: tripleoupstream/centos-binary-cinder-api:latest
-      image_pull_policy: IfNotPresent
-#      command: ["/usr/bin/cinder-manage", "db", "sync"]
-      env:
-        - name: KOLLA_KUBERNETES
-          value: ""
-        - name: KOLLA_BOOTSTRAP
-          value: ""
-        - name: KOLLA_CONFIG_STRATEGY
-          value: "COPY_ONCE"
-        - name: "TZ"
-          value: "UTC"
-      volume_mounts:
-        - name: kolla-config
-          mountPath: /var/lib/kolla/config_files/
-    volumes:
-      - name: kolla-config
-        config_map:
-          name: cinder
-  when: result|succeeded
-
-- name: create cinder-api service
-  k8s_v1_service:
-    name: cinder
-    host: "{{coe_host}}"
-    context: "{{kube_context}}"
-    kubeconfig: "{{config_file}}"
-    namespace: "{{ namespace }}"
-    labels:
-      app: cinder-api
-      service: cinder-api
-    selector:
-      app: cinder-api
-      service: cinder-api
-    ports:
-      - name: cinder-api
-        port: 8776
-        target_port: 8776
-
-# Cinder API
-- name: Create Cinder-api Deployment
-  k8s_v1beta1_deployment:
-    name: cinder-api
-    host: "{{coe_host}}"
-    context: "{{kube_context}}"
-    kubeconfig: "{{config_file}}"
-    namespace: "{{ namespace }}"
-    service_account_name: "{{ service_account }}"
-    labels:
-      app: cinder-api
-      service: cinder-api
-    replicas: 1
-    spec_template_metadata_labels:
-      app: cinder-api
-      service: cinder-api
-    containers:
-    - name: cinder-api
-      image: tripleoupstream/centos-binary-cinder-api:latest
-      ports:
-        - container_port: 8776
-          protocol: TCP
-      env:
-        - name: TZ
-          value: UTC
-        - name: KOLLA_CONFIG_STRATEGY
-          value: COPY_ALWAYS
-        - name: KOLLA_KUBERNETES
-          value: ""
-      volume_mounts:
-        - name: kolla-config
-          mountPath: /var/lib/kolla/config_files/
-    volumes:
-      - name: kolla-config
-        config_map:
-          name: cinder-api
-
-# Cinder Scheduler
-- name: create cinder-scheduler deployment
-  k8s_v1beta1_deployment:
-    name: cinder-scheduler
-    host: "{{coe_host}}"
-    context: "{{kube_context}}"
-    kubeconfig: "{{config_file}}"
-    namespace: "{{ namespace }}"
-    service_account_name: "{{ service_account }}"
-    labels:
-      app: cinder-scheduler
-      service: cinder-scheduler
-    replicas: 1
-    spec_template_metadata_labels:
-      app: cinder-scheduler
-      service: cinder-scheduler
-    containers:
-    - name: cinder-scheduler
-      image: tripleoupstream/centos-binary-cinder-scheduler:latest
-      env:
-        - name: TZ
-          value: UTC
-        - name: KOLLA_CONFIG_STRATEGY
-          value: COPY_ALWAYS
-        - name: KOLLA_KUBERNETES
-          value: ""
-      volume_mounts:
-        - name: kolla-config
-          mountPath: /var/lib/kolla/config_files/
-    volumes:
-      - name: kolla-config
-        config_map:
-          name: cinder-scheduler
-
-- name: Create Cinder Volume Deployment
-  k8s_v1beta1_deployment:
-    name: cinder-volume
-    host: "{{coe_host}}"
-    context: "{{kube_context}}"
-    kubeconfig: "{{config_file}}"
-    namespace: "{{ namespace }}"
-    service_account_name: "{{ service_account }}"
-    labels:
-      app: cinder-volume
-      service: cinder-volume
-    replicas: 1
-    spec_template_metadata_labels:
-      app: cinder-volume
-      service: cinder-volume
-    containers:
-    - name: cinder-volume
-      image: tripleoupstream/centos-binary-cinder-volume:latest
-      image_pull_policy: IfNotPresent
-      volumeMounts:
-        - name: kolla-config
-          mountPath: /var/lib/kolla/config_files/
-      env:
-        - name: TZ
-          value: UTC
-        - name: KOLLA_CONFIG_STRATEGY
-          value: COPY_ALWAYS
-        - name: KOLLA_KUBERNETES
-          value: ""
-    volumes:
-      - name: kolla-config
-        config_map:
-          name: cinder-volume

tasks/test.yml

@@ -0,0 +1,47 @@
+- name: Initialize an empty list for our backends
+  set_fact:
+    enabled_backends: []
+
+- include: hiera.yml
+
+- include: enabled_backends.yml
+
+- name: Generate config files
+  config_template:
+    src: cinder.conf.j2
+    dest: /tmp/cinder.conf
+    config_overrides: '{{cinder_config}}'
+    config_type: ini
+
+- name: Generate config files
+  template:
+    src: cinder-volume-config.json.j2
+    dest: /tmp/cinder-volume-config.json
+
+- name: Read configs into memory
+  slurp:
+    src: "/tmp/cinder.conf"
+  register: "cinder_conf"
+
+- name: Read configs into memory
+  slurp:
+    src: "/tmp/cinder-volume-config.json"
+  register: "cinder_volume_config_json"
+
+- name: Create cinder configmaps
+  ignore_errors: yes
+  k8s_v1_config_map:
+    name: cinder-api
+    namespace: "{{ namespace }}"
+    state: present
+    debug: yes
+    labels:
+      service: cinder
+    data:
+      config.json: |
+        {{cinder_volume_config_json['content'] | b64decode | to_json}}
+
+      cinder.conf: |
+        {{cinder_conf['content'] | b64decode}}
+
+

templates/cinder-volume-config.yml.j2

@@ -0,0 +1,30 @@
+command: "/usr/bin/cinder-volume --config-file /usr/share/cinder/cinder-dist.conf --config-file /etc/cinder/cinder.conf"
+config_files:
+- source: "/var/lib/kolla/config_files/cinder.conf"
+  dest: "/etc/cinder/cinder.conf"
+  owner: cinder
+  perm: '0600'
+- source: "/var/lib/kolla/config_files/ceph.conf"
+  dest: "/etc/ceph/ceph.conf"
+  owner: root
+  perm: '0644'
+{% if result.conf_dict['cinder_enable_rbd_backend'] %}
+- source: "/var/lib/kolla/config_files/ceph.client.{{result.conf_dict['cinder_rbd_user_name']}}.keyring"
+  dest: "/etc/ceph/ceph.client.{{result.conf_dict['cinder_rbd_user_name']}}.keyring"
+  owner: root
+  perm: '0644'
+{% endif %}
+{% if result.conf_dict['cinder_enable_iscsi_backend'] %}
+- source: "/var/lib/kolla/config_files/lvm.conf"
+  dest: "/etc/lvm/lvm.conf"
+  owner: root
+  perm: '0644'
+{% endif %}
+permissions:
+- path: "/var/lib/cinder"
+  owner: cinder:cinder
+  recurse: true
+- path: "/var/log/kolla/cinder"
+  owner: cinder:cinder
+  recurse: true
+

templates/cinder.conf.j2

@@ -5,14 +5,15 @@ enable_v3_api=True
 storage_availability_zone=nova
 default_availability_zone=nova
 auth_strategy=noauth
-enabled_backends=tripleo_iscsi
+enabled_backends={{ enabled_backends | join(",") }}
 nova_catalog_info=compute:nova:internalURL
 nova_catalog_admin_info=compute:nova:adminURL
 scheduler_driver=cinder.scheduler.filter_scheduler.FilterScheduler
 osapi_volume_listen=0.0.0.0
 osapi_volume_workers=2
 log_dir=/var/log/cinder
-transport_url=rabbit://guest:rabbitpass@rabbitmq:5672/?ssl=0
+#transport_url=rabbit://guest:rabbitpass@rabbitmq:5672/?ssl=0
+transport_url=rabbit://{{result.conf_dict['oslo_messaging_rabbit']['rabbit_userid']}}:{{result.conf_dict['oslo_messaging_rabbit']['rabbit_password']}}@rabbitmq:{{result.conf_dict['oslo_messaging_rabbit']['rabbit_port']}}/?ssl=0
 rpc_backend=rabbit
 control_exchange=openstack
 api_paste_config=/etc/cinder/api-paste.ini
@@ -27,7 +28,7 @@ lock_path=/var/lib/cinder/tmp
 
 [oslo_messaging_notifications]
 driver=messagingv2
-transport_url=rabbit://guest:rabbitpass@rabbitmq:5672/?ssl=0
+transport_url=rabbit://{{result.conf_dict['oslo_messaging_rabbit']['rabbit_userid']}}:{{result.conf_dict['oslo_messaging_rabbit']['rabbit_password']}}@rabbitmq:{{result.conf_dict['oslo_messaging_rabbit']['rabbit_port']}}/?ssl=0
 
 [oslo_messaging_rabbit]
 ssl=False
@@ -42,6 +43,7 @@ enable_proxy_headers_parsing=True
 [oslo_policy]
 policy_file=/etc/cinder/policy.json
 
+{% if result.conf_dict['cinder_enable_iscsi_backend'] %}
 [tripleo_iscsi]
 volume_backend_name=tripleo_iscsi
 volume_driver=cinder.volume.drivers.lvm.LVMVolumeDriver
@@ -49,4 +51,28 @@ iscsi_ip_address=iscsid
 iscsi_helper=lioadm
 volumes_dir=/var/lib/cinder/cinder-volumes
 iscsi_protocol=iscsi
+{% endif %}
 
+{% if result.conf_dict['dellemc_xtremio_iscsi_backend'] %}
+[xtremio]
+max_over_subscription_ratio = 40.0
+use_multipath_for_image_xfer=true
+volume_backend_name=xtremio
+volume_driver=cinder.volume.drivers.emc.xtremio.XtremIOISCSIDriver
+san_ip=10.35.146.7
+xtremio_cluster_name=RHEL-QE
+san_login=cinder
+san_password=qum5net
+image_volume_cache_enabled = false
+{% endif %}
+
+{% if result.conf_dict['cinder_enable_rbd_backend'] %}
+[tripleo_ceph]
+volume_backend_name=tripleo_ceph
+volume_driver=cinder.volume.drivers.rbd.RBDDriver
+rbd_ceph_conf=/etc/ceph/ceph.conf
+rbd_user={{ result.conf_dict['cinder_rbd_user_name'] }}
+rbd_pool={{ result.conf_dict['cinder_rbd_pool_name'] }}
+rbd_secret_uuid={{ result.conf_dict['ceph_cluster_fsid'] }}
+backend_host=hostgroup
+{% endif %}

templates/rbd/ceph.conf.j2

@@ -0,0 +1,10 @@
+[global]
+fsid = 
+mon_host = 
+auth_cluster_required = cephx
+auth_service_required = cephx
+auth_client_required = cephx
+auth_supported = cephx
+
+[client]
+rbd_default_features = 1

templates/rbd/tripleo-ceph.conf.j2

@@ -0,0 +1,8 @@
+[tripleo_ceph]
+volume_backend_name=tripleo_ceph
+volume_driver=cinder.volume.drivers.rbd.RBDDriver
+rbd_ceph_conf=/etc/ceph/ceph.conf
+rbd_user=wshed
+rbd_pool=wshed_volumes
+rbd_secret_uuid=cd0d33e2-9e44-489d-8b5c-53e4fcb9a8ad
+backend_host=hostgroup

templates/xtremio.conf

@@ -0,0 +1,10 @@
+[xtremio]
+max_over_subscription_ratio = 40.0
+use_multipath_for_image_xfer=true
+volume_backend_name=xtremio
+volume_driver=cinder.volume.drivers.emc.xtremio.XtremIOISCSIDriver
+san_ip=10.35.146.7
+xtremio_cluster_name=RHEL-QE
+san_login=cinder
+san_password=qum5net
+image_volume_cache_enabled = false