Merge "[Database] Shipyard DB changes"

This commit is contained in:
Zuul 2019-03-07 21:21:39 +00:00 committed by Gerrit Code Review
commit 38956c7b27
12 changed files with 193 additions and 146 deletions

View File

@ -1,50 +0,0 @@
#!/bin/bash
{{/*
Copyright 2017 The Openstack-Helm Authors.
Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
set -ex
export HOME=/tmp
# Extract the DB string from shipyard.conf and get the
# value of the DB host and port
db_string=`grep -i sql_alchemy_conn ${AIRFLOW_CONFIG_FILE}`
db_fqdn=`echo ${db_string#*@} | cut -f1 -d"."`
db_port=`echo ${db_string#*@} | grep -o "[0-9]\+"`
pgsql_superuser_cmd () {
DB_COMMAND="$1"
if [[ ! -z $2 ]]; then
EXPORT PGDATABASE=$2
fi
psql \
-h $db_fqdn \
-p $db_port \
-U ${DB_ADMIN_USER} \
--command="${DB_COMMAND}"
}
# Create db
pgsql_superuser_cmd "SELECT 1 FROM pg_database WHERE datname = '$DB_NAME'" | grep -q 1 || pgsql_superuser_cmd "CREATE DATABASE $DB_NAME"
# Create db user
pgsql_superuser_cmd "SELECT * FROM pg_roles WHERE rolname = '$DB_SERVICE_USER';" | tail -n +3 | head -n -2 | grep -q 1 || \
pgsql_superuser_cmd "CREATE ROLE ${DB_SERVICE_USER} LOGIN PASSWORD '$DB_SERVICE_PASSWORD'"
# Grant permissions to user
pgsql_superuser_cmd "GRANT ALL PRIVILEGES ON DATABASE $DB_NAME to $DB_SERVICE_USER;"

View File

@ -0,0 +1,46 @@
#!/bin/bash
{{/*
Copyright (c) 2019 AT&T Intellectual Property. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
set -e
if [[ ! -v DB_HOST ]]; then
echo "environment variable DB_HOST not set"
exit 1
elif [[ ! -v DB_ADMIN_USER ]]; then
echo "environment variable DB_ADMIN_USER not set"
exit 1
elif [[ ! -v PGPASSWORD ]]; then
echo "environment variable PGPASSWORD not set"
exit 1
elif [[ ! -v USER_DB_USER ]]; then
echo "environment variable USER_DB_USER not set"
exit 1
elif [[ ! -v DB_PORT ]]; then
echo "environment variable USER_DB_USER not set"
exit 1
elif [[ ! -v USER_DB_NAME ]]; then
echo "environment variable USER_DB_NAME not set"
exit 1
else
echo "Got DB connection info"
fi
# Grant permissions to shipyard user to the airflow dataabase tables
# This will allow shipyard user to query airflow database
/usr/bin/psql -h ${DB_HOST} -p ${DB_PORT} -U ${DB_ADMIN_USER} -d ${AIRFLOW_DB_NAME} \
--command="GRANT select, insert, update, delete on all tables in schema public to $USER_DB_USER;"

View File

@ -1,55 +0,0 @@
#!/bin/bash
{{/*
Copyright 2017 The Openstack-Helm Authors.
Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
set -ex
export HOME=/tmp
# Extract the DB string from shipyard.conf and get the
# value of the DB host and port
db_string=`grep -i postgresql_db ${SHIPYARD_CONFIG_FILE}`
db_fqdn=`echo ${db_string#*@} | cut -f1 -d"."`
db_port=`echo ${db_string#*@} | grep -o "[0-9]\+"`
pgsql_superuser_cmd () {
DB_COMMAND="$1"
if [[ ! -z $2 ]]; then
EXPORT PGDATABASE=$2
fi
psql \
-h $db_fqdn \
-p $db_port \
-U ${DB_ADMIN_USER} \
--command="${DB_COMMAND}"
}
# Create db
pgsql_superuser_cmd "SELECT 1 FROM pg_database WHERE datname = '$DB_NAME'" | grep -q 1 || pgsql_superuser_cmd "CREATE DATABASE $DB_NAME"
# Create db user
pgsql_superuser_cmd "SELECT * FROM pg_roles WHERE rolname = '$DB_SERVICE_USER';" | tail -n +3 | head -n -2 | grep -q 1 || \
pgsql_superuser_cmd "CREATE ROLE ${DB_SERVICE_USER} LOGIN PASSWORD '$DB_SERVICE_PASSWORD'"
# Grant permissions to user
pgsql_superuser_cmd "GRANT ALL PRIVILEGES ON DATABASE $DB_NAME to $DB_SERVICE_USER;"
# Grant permissions to shipyard user to the airflow dataabase tables
# This will allow shipyard user to query airflow database
psql -h $db_fqdn -p $db_port -U ${DB_ADMIN_USER} -d ${AIRFLOW_DB_NAME} \
--command="GRANT select, insert, update, delete on all tables in schema public to $DB_SERVICE_USER;"

View File

@ -24,7 +24,7 @@ data:
rabbit-init.sh: |+
{{- include "helm-toolkit.scripts.rabbit_init" . | indent 4 }}
airflow-db-init.sh: |+
{{ tuple "bin/_airflow-db-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{- include "helm-toolkit.scripts.pg_db_init" . | indent 4 }}
airflow-db-sync.sh: |+
{{ tuple "bin/_airflow-db-sync.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{- end }}

View File

@ -28,7 +28,9 @@ data:
ks-domain-user.sh: |
{{- include "helm-toolkit.scripts.keystone_domain_user" . | indent 4 }}
shipyard-db-init.sh: |+
{{ tuple "bin/_shipyard-db-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{- include "helm-toolkit.scripts.pg_db_init" . | indent 4 }}
shipyard-db-sync.sh: |+
{{ tuple "bin/_shipyard-db-sync.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
shipyard-db-auxiliary.sh: |+
{{ tuple "bin/_shipyard-db-auxiliary.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{- end }}

View File

@ -44,24 +44,22 @@ spec:
imagePullPolicy: {{ .Values.images.pull_policy | quote }}
{{ tuple $envAll $envAll.Values.pod.resources.jobs.airflow_db_init | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
env:
- name: AIRFLOW_DB_URL
- name: DB_HOST
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_airflow_db.user }}
key: AIRFLOW_DATABASE_URI
- name: AIRFLOW_CONFIG_FILE
value: /etc/airflow/airflow.cfg
- name: DB_NAME
key: AIRFLOW_DATABASE_HOST
- name: USER_DB_NAME
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_airflow_db.user }}
key: AIRFLOW_DATABASE_NAME
- name: DB_SERVICE_USER
- name: USER_DB_USER
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_airflow_db.user }}
key: AIRFLOW_DATABASE_USERNAME
- name: DB_SERVICE_PASSWORD
- name: USER_DB_PASS
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_airflow_db.user }}
@ -76,6 +74,11 @@ spec:
secretKeyRef:
name: {{ .Values.secrets.postgresql_airflow_db.admin }}
key: AIRFLOW_DATABASE_PASSWORD
- name: DB_PORT
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_airflow_db.user }}
key: DATABASE_PORT
command:
- /tmp/airflow-db-init.sh
volumeMounts:

View File

@ -0,0 +1,99 @@
{{/*
Copyright (c) 2019 AT&T Intellectual Property. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.job_shipyard_db_auxiliary }}
{{- $envAll := . }}
{{- $mounts_shipyard_db_auxiliary := .Values.pod.mounts.shipyard_db_auxiliary.shipyard_db_auxiliary }}
{{- $mounts_shipyard_db_auxiliary_init := .Values.pod.mounts.shipyard_db_auxiliary.init_container }}
{{- $serviceAccountName := "shipyard-db-auxiliary" }}
{{ tuple $envAll "shipyard_db_auxiliary" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
---
apiVersion: batch/v1
kind: Job
metadata:
name: shipyard-db-auxiliary
annotations:
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
spec:
template:
metadata:
labels:
{{ tuple $envAll "shipyard" "db-auxiliary" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
spec:
serviceAccountName: {{ $serviceAccountName }}
restartPolicy: OnFailure
nodeSelector:
{{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
initContainers:
{{ tuple $envAll "shipyard_db_auxiliary" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
containers:
- name: shipyard-db-auxiliary
image: {{ .Values.images.tags.shipyard_db_auxiliary | quote }}
imagePullPolicy: {{ .Values.images.pull_policy | quote }}
{{ tuple $envAll $envAll.Values.pod.resources.jobs.shipyard_db_auxiliary | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
env:
- name: DB_HOST
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_shipyard_db.user }}
key: SHIPYARD_DATABASE_HOST
- name: USER_DB_NAME
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_shipyard_db.user }}
key: SHIPYARD_DATABASE_NAME
- name: USER_DB_USER
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_shipyard_db.user }}
key: SHIPYARD_DATABASE_USERNAME
- name: DB_ADMIN_USER
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_shipyard_db.admin }}
key: SHIPYARD_DATABASE_USERNAME
- name: PGPASSWORD
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_shipyard_db.admin }}
key: SHIPYARD_DATABASE_PASSWORD
- name: AIRFLOW_DB_NAME
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_airflow_db.user }}
key: AIRFLOW_DATABASE_NAME
- name: DB_PORT
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_shipyard_db.user }}
key: DATABASE_PORT
- name: SHIPYARD_AUXILIARY_CONFIG
value: "True"
command:
- /tmp/shipyard-db-auxiliary.sh
volumeMounts:
- name: shipyard-bin
mountPath: /tmp/shipyard-db-auxiliary.sh
subPath: shipyard-db-auxiliary.sh
readOnly: true
{{ if $mounts_shipyard_db_auxiliary.volumeMounts }}{{ toYaml $mounts_shipyard_db_auxiliary.volumeMounts | indent 10 }}{{ end }}
volumes:
- name: shipyard-bin
configMap:
name: shipyard-bin
defaultMode: 0555
{{ if $mounts_shipyard_db_auxiliary.volumes }}{{ toYaml $mounts_shipyard_db_auxiliary.volumes | indent 6 }}{{ end }}
{{- end }}

View File

@ -46,24 +46,22 @@ spec:
imagePullPolicy: {{ .Values.images.pull_policy | quote }}
{{ tuple $envAll $envAll.Values.pod.resources.jobs.shipyard_db_init | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
env:
- name: SHIPYARD_DB_URL
- name: DB_HOST
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_shipyard_db.user }}
key: SHIPYARD_DATABASE_URI
- name: SHIPYARD_CONFIG_FILE
value: /etc/shipyard/shipyard.conf
- name: DB_NAME
key: SHIPYARD_DATABASE_HOST
- name: USER_DB_NAME
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_shipyard_db.user }}
key: SHIPYARD_DATABASE_NAME
- name: DB_SERVICE_USER
- name: USER_DB_USER
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_shipyard_db.user }}
key: SHIPYARD_DATABASE_USERNAME
- name: DB_SERVICE_PASSWORD
- name: USER_DB_PASS
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_shipyard_db.user }}
@ -83,6 +81,13 @@ spec:
secretKeyRef:
name: {{ .Values.secrets.postgresql_airflow_db.user }}
key: AIRFLOW_DATABASE_NAME
- name: DB_PORT
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_shipyard_db.user }}
key: DATABASE_PORT
- name: SHIPYARD_AUXILIARY_CONFIG
value: "True"
command:
- /tmp/shipyard-db-init.sh
volumeMounts:

View File

@ -46,28 +46,8 @@ spec:
imagePullPolicy: {{ .Values.images.pull_policy }}
{{ tuple $envAll $envAll.Values.pod.resources.jobs.shipyard_db_sync | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
env:
- name: SHIPYARD_DB_URL
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_shipyard_db.user }}
key: SHIPYARD_DATABASE_URI
- name: SHIPYARD_CONFIG_FILE
value: /etc/shipyard/shipyard.conf
- name: DB_NAME
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_shipyard_db.user }}
key: SHIPYARD_DATABASE_NAME
- name: DB_SERVICE_USER
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_shipyard_db.user }}
key: SHIPYARD_DATABASE_USERNAME
- name: DB_SERVICE_PASSWORD
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_shipyard_db.user }}
key: SHIPYARD_DATABASE_PASSWORD
command:
- /tmp/shipyard-db-sync.sh
volumeMounts:

View File

@ -35,6 +35,9 @@ data:
{{ $auth.username | b64enc | indent 4 }}
AIRFLOW_DATABASE_URI: |-
{{ tuple "postgresql_airflow_db" "internal" "user" "postgresql" $envAll | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" | b64enc | indent 4 }}
AIRFLOW_DATABASE_HOST: |-
{{ tuple "postgresql_airflow_db" "internal" $envAll | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" | b64enc | indent 4 }}
DATABASE_PORT: {{ tuple "postgresql_airflow_db" "internal" "postgresql" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | b64enc }}
...
{{- end }}
{{- end }}

View File

@ -33,8 +33,9 @@ data:
{{ $auth.password | b64enc | indent 4 }}
SHIPYARD_DATABASE_USERNAME: |-
{{ $auth.username | b64enc | indent 4 }}
SHIPYARD_DATABASE_URI: |-
{{ tuple "postgresql_shipyard_db" "internal" "user" "postgresql" $envAll | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" | b64enc | indent 4 }}
SHIPYARD_DATABASE_HOST: |-
{{ tuple "postgresql_shipyard_db" "internal" $envAll | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" | b64enc | indent 4 }}
DATABASE_PORT: {{ tuple "postgresql_shipyard_db" "internal" "postgresql" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | b64enc }}
...
{{- end }}
{{- end }}

View File

@ -32,6 +32,7 @@ images:
shipyard: quay.io/airshipit/shipyard:latest
dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
shipyard_db_init: docker.io/postgres:9.5
shipyard_db_auxiliary: docker.io/postgres:9.5
shipyard_db_sync: quay.io/airshipit/shipyard:latest
airflow_db_init: docker.io/postgres:9.5
rabbit_init: docker.io/rabbitmq:3.7-management
@ -79,9 +80,12 @@ dependencies:
services:
- service: postgresql_shipyard_db
endpoint: internal
shipyard_db_sync:
shipyard_db_auxiliary:
jobs:
- shipyard-db-init
shipyard_db_sync:
jobs:
- shipyard-db-auxiliary
services:
- service: postgresql_shipyard_db
endpoint: internal
@ -115,7 +119,6 @@ dependencies:
endpoint: internal
shipyard:
jobs:
- shipyard-db-init
- shipyard-db-sync
- shipyard-ks-endpoints
- shipyard-ks-user
@ -128,7 +131,6 @@ dependencies:
airflow_server:
jobs:
- airflow-rabbit-init
- airflow-db-init
- airflow-db-sync
services:
- service: postgresql_airflow_db
@ -691,6 +693,9 @@ pod:
shipyard_db_init:
init_container: null
shipyard_db_init:
shipyard_db_auxiliary:
init_container: null
shipyard_db_auxiliary:
shipyard_db_sync:
init_container: null
shipyard_db_sync:
@ -801,6 +806,13 @@ pod:
requests:
memory: "128Mi"
cpu: "500m"
shipyard_db_auxiliary:
limits:
memory: "128Mi"
cpu: "500m"
requests:
memory: "128Mi"
cpu: "500m"
shipyard_db_sync:
limits:
memory: "128Mi"
@ -829,6 +841,7 @@ manifests:
statefulset_airflow_worker: true
ingress_shipyard_api: true
job_shipyard_db_init: true
job_shipyard_db_auxiliary: true
job_shipyard_db_sync: true
job_rabbit_init: true
job_airflow_db_init: true