Initial implementation of sahara

* sahara-api added
* sahara-engine added

Change-Id: I964d49dc3eea9a678da95bd60b2f63b8a205b2c2

.gitignore

@@ -0,0 +1 @@
+.tox

docker/sahara-api/Dockerfile.j2

@@ -0,0 +1,9 @@
+FROM {{ image_spec("sahara-base") }}
+MAINTAINER {{ maintainer }}
+
+#mysql-client only for provisioning need to be removed later
+RUN apt-get install -y --no-install-recommends \
+        mysql-client \
+    && apt-get clean
+
+USER sahara

docker/sahara-base/Dockerfile.j2

@@ -0,0 +1,11 @@
+FROM {{ image_spec("openstack-base") }}
+MAINTAINER {{ maintainer }}
+
+{{ copy_sources("openstack/sahara", "/sahara") }}
+
+RUN apt update && apt install -y openssh-client && apt-get clean \
+    && useradd --user-group sahara \
+    && /var/lib/microservices/venv/bin/pip install --upgrade pymemcache /sahara \
+    && mkdir -p /etc/sahara \
+    && cp -r /sahara/etc/sahara/* /etc/sahara \
+    && chown -R sahara:sahara /etc/sahara

docker/sahara-engine/Dockerfile.j2

@@ -0,0 +1,4 @@
+FROM {{ image_spec("sahara-base") }}
+MAINTAINER {{ maintainer }}
+
+USER sahara

service/files/defaults.yaml

@@ -0,0 +1,29 @@
+configs:
+  sahara:
+    debug: false
+    port:
+      cont: 8386
+      ingress: data-processing
+
+    plugins:
+      - fake
+      - vanilla
+      - spark
+      - cdh
+      - ambari
+      - storm
+      - mapr
+
+    auth:
+      user: sahara
+      password: password
+
+    db:
+      username: sahara
+      password: password
+      name: sahara
+
+sources:
+  openstack/sahara:
+    git_url: https://git.openstack.org/openstack/sahara.git
+    git_ref: stable/newton

service/files/sahara.conf.j2

@@ -0,0 +1,37 @@
+[DEFAULT]
+debug = {{ sahara.debug }}
+
+use_syslog = false
+use_stderr = true
+
+use_neutron = true
+plugins = {{ sahara.plugins | join(',') }}
+
+host = {{ network_topology["private"]["address"] }}
+port = {{ sahara.port.cont }}
+
+periodic_coordinator_backend_url = memcached://{{ address("memcached", memcached.port) }}
+transport_url = rabbit://{{ rabbitmq.user }}:{{ rabbitmq.password }}@{{ address("rabbitmq", rabbitmq.port) }}/
+
+[database]
+connection = mysql+pymysql://{{ sahara.db.username }}:{{ sahara.db.password }}@{{ address(service.database) }}/{{ sahara.db.name }}
+max_retries = -1
+
+[keystone_authtoken]
+auth_uri = {{ address("keystone", keystone.public_port, with_scheme=True) }}
+auth_url = {{ address("keystone", keystone.admin_port, with_scheme=True) }}
+auth_type = password
+project_domain_id = default
+user_domain_id = default
+project_name = {{ openstack.project_name }}
+username = {{ sahara.auth.user }}
+password = {{ sahara.auth.password }}
+memcached_servers = {{ address("memcached", memcached.port) }}
+
+# sahara trustor user
+admin_password = {{ sahara.auth.password }}
+admin_user = {{ sahara.auth.user }}
+admin_tenant_name = {{ openstack.project_name }}
+
+[oslo_messaging_notifications]
+driver = noop

service/sahara-api.yaml

@@ -0,0 +1,65 @@
+dsl_version: 0.1.0
+service:
+  name: sahara-api
+  ports:
+    - {{ sahara.port }}
+  containers:
+    - name: sahara-api
+      image: sahara-api
+      # TODO(sreshetniak): add probes
+      pre:
+        - name: sahara-db-create
+          dependencies:
+            - {{ service.database }}
+          type: single
+          command:
+            mysql -u root -p{{ db.root_password }} -h {{ address(service.database) }} -e "create database {{ sahara.db.name }};
+            grant all privileges on {{ sahara.db.name }}.* to '{{ sahara.db.username }}'@'%' identified by '{{ sahara.db.password }}';"
+        - name: sahara-db-sync
+          files:
+            - sahara-conf
+          dependencies:
+            - sahara-db-create
+          type: single
+          command: sahara-db-manage --config-file /etc/sahara/sahara.conf upgrade head
+        - name: sahara-user-create
+          dependencies:
+            - keystone
+          type: single
+          command: openstack user create --domain default --password {{ sahara.auth.password  }} {{ sahara.auth.user }}
+        - name: sahara-role-add
+          dependencies:
+            - sahara-user-create
+          type: single
+          command: openstack role add --project {{ openstack.project_name }} --user {{ sahara.auth.user }} admin
+        - name: sahara-service-create
+          dependencies:
+            - keystone
+          type: single
+          command: openstack service create --name sahara --description "OpenStack Data Processing service" data-processing
+        - name: sahara-public-endpoint-create
+          dependencies:
+            - sahara-service-create
+          type: single
+          command: openstack endpoint create --region RegionOne data-processing public {{ address("sahara-api", sahara.port, external=True, with_scheme=True) }}/v1.1/\$\(tenant_id\)s
+        - name: sahara-internal-endpoint-create
+          dependencies:
+            - sahara-service-create
+          type: single
+          command: openstack endpoint create --region RegionOne data-processing internal http://{{ address("sahara-api", sahara.port, with_scheme=True) }}/v1.1/\$\(tenant_id\)s
+        - name: sahara-admin-endpoint-create
+          dependencies:
+            - sahara-service-create
+          type: single
+          command: openstack endpoint create --region RegionOne data-processing admin http://{{ address("sahara-api", sahara.port, with_scheme=True) }}/v1.1/\$\(tenant_id\)s
+      daemon:
+        files:
+          - sahara-conf
+        dependencies:
+          - rabbitmq
+        command: sahara-api --config-file /etc/sahara/sahara.conf
+
+files:
+  sahara-conf:
+    path: /etc/sahara/sahara.conf
+    content: sahara.conf.j2

service/sahara-engine.yaml

@@ -0,0 +1,19 @@
+dsl_version: 0.1.0
+service:
+  name: sahara-engine
+  containers:
+    - name: sahara-engine
+      image: sahara-engine
+      # TODO(sreshetniak): add probes
+      daemon:
+        files:
+          - sahara-conf
+        dependencies:
+          - rabbitmq
+          - sahara-api
+        command: sahara-engine --config-file /etc/sahara/sahara.conf
+
+files:
+  sahara-conf:
+    path: /etc/sahara/sahara.conf
+    content: sahara.conf.j2

test-requirements.txt

@@ -0,0 +1,2 @@
+yamllint>=1.3.2
+bashate>=0.2

tools/yamllint.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -ex
+
+workdir=$(dirname $0)
+yamllint -c $workdir/yamllint.yaml $(find . -not -path '*/\.*' -type f -name '*.yaml')

tools/yamllint.yaml

@@ -0,0 +1,21 @@
+extends: default
+
+rules:
+  braces:
+    max-spaces-inside: 1
+  comments:
+    level: error
+  comments-indentation:
+    level: warning
+  document-end:
+    present: no
+  document-start:
+    level: error
+    present: no
+  empty-lines:
+    max: 1
+    max-start: 0
+    max-end: 0
+  line-length:
+    level: warning
+    max: 120

tox.ini

@@ -0,0 +1,15 @@
+[tox]
+minversion = 1.6
+envlist = linters,bashate
+skipsdist = True
+
+[testenv]
+deps = -r{toxinidir}/test-requirements.txt
+
+[testenv:linters]
+commands =
+    {toxinidir}/tools/yamllint.sh
+
+[testenv:bashate]
+whitelist_externals = bash
+commands = bash -c "find {toxinidir} -type f -name '*.sh' -not -path '*/.tox/*' -print0 | xargs -0 bashate -v"