#!/bin/bash # # lib/gyan # Functions to control the configuration and operation of the **gyan** service # Dependencies: # # - ``functions`` file # - ``DEST``, ``DATA_DIR``, ``STACK_USER`` must be defined # - ``SERVICE_{TENANT_NAME|PASSWORD}`` must be defined # ``stack.sh`` calls the entry points in this order: # # - install_gyan # - configure_gyan # - create_gyan_conf # - create_gyan_accounts # - init_gyan # - start_gyan # - stop_gyan # - cleanup_gyan # Save trace setting XTRACE=$(set +o | grep xtrace) set +o xtrace # Defaults # -------- # Set up default directories GYAN_REPO=${GYAN_REPO:-${GIT_BASE}/openstack/gyan.git} GYAN_BRANCH=${GYAN_BRANCH:-master} GYAN_DIR=$DEST/gyan GITREPO["python-gyanclient"]=${GYANCLIENT_REPO:-${GIT_BASE}/openstack/python-gyanclient.git} GITBRANCH["python-gyanclient"]=${GYANCLIENT_BRANCH:-master} GITDIR["python-gyanclient"]=$DEST/python-gyanclient GYAN_STATE_PATH=${GYAN_STATE_PATH:=$DATA_DIR/gyan} GYAN_AUTH_CACHE_DIR=${GYAN_AUTH_CACHE_DIR:-/var/cache/gyan} GYAN_CONF_DIR=/etc/gyan GYAN_CONF=$GYAN_CONF_DIR/gyan.conf GYAN_API_PASTE=$GYAN_CONF_DIR/api-paste.ini if is_ssl_enabled_service "gyan" || is_service_enabled tls-proxy; then GYAN_SERVICE_PROTOCOL="https" fi # Toggle for deploying GYAN-API under a wsgi server GYAN_USE_UWSGI=${GYAN_USE_UWSGI:-True} # Public facing bits GYAN_SERVICE_HOST=${GYAN_SERVICE_HOST:-$SERVICE_HOST} GYAN_SERVICE_PORT=${GYAN_SERVICE_PORT:-8517} GYAN_SERVICE_PORT_INT=${GYAN_SERVICE_PORT_INT:-18517} GYAN_SERVICE_PROTOCOL=${GYAN_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL} GYAN_TRUSTEE_DOMAIN_ADMIN_PASSWORD=${GYAN_TRUSTEE_DOMAIN_ADMIN_PASSWORD:-secret} # Support entry points installation of console scripts if [[ -d $GYAN_DIR/bin ]]; then GYAN_BIN_DIR=$GYAN_DIR/bin else GYAN_BIN_DIR=$(get_python_exec_prefix) fi GYAN_UWSGI=$GYAN_BIN_DIR/gyan-api-wsgi GYAN_UWSGI_CONF=$GYAN_CONF_DIR/gyan-api-uwsgi.ini GYAN_DB_TYPE=${GYAN_DB_TYPE:-sql} if is_ubuntu; then UBUNTU_RELEASE_BASE_NUM=`lsb_release -r | awk '{print $2}' | cut -d '.' -f 1` fi # Functions # --------- # cleanup_gyan() - Remove residual data files, anything left over from previous # runs that a clean run would need to clean up function cleanup_gyan { sudo rm -rf $GYAN_STATE_PATH $GYAN_AUTH_CACHE_DIR remove_uwsgi_config "$GYAN_UWSGI_CONF" "$GYAN_UWSGI" } # configure_gyan() - Set config files, create data dirs, etc function configure_gyan { # Put config files in ``/etc/gyan`` for everyone to find if [[ ! -d $GYAN_CONF_DIR ]]; then sudo mkdir -p $GYAN_CONF_DIR sudo chown $STACK_USER $GYAN_CONF_DIR fi configure_rootwrap gyan # Rebuild the config file from scratch create_gyan_conf create_api_paste_conf write_uwsgi_config "$GYAN_UWSGI_CONF" "$GYAN_UWSGI" "/ml-infra" if [[ "$USE_PYTHON3" = "True" ]]; then # Switch off glance->swift communication as swift fails under py3.x iniset /etc/glance/glance-api.conf glance_store default_store file fi } # create_gyan_accounts() - Set up common required GYAN accounts # # Project User Roles # ------------------------------------------------------------------ # SERVICE_PROJECT_NAME gyan service function create_gyan_accounts { create_service_user "gyan" "admin" if is_service_enabled gyan-api; then local gyan_api_url if [[ "$GYAN_USE_UWSGI" == "True" ]]; then gyan_api_url="$GYAN_SERVICE_PROTOCOL://$GYAN_SERVICE_HOST/ml-infra" else gyan_api_url="$GYAN_SERVICE_PROTOCOL://$GYAN_SERVICE_HOST:$GYAN_SERVICE_PORT" fi local gyan_service=$(get_or_create_service "gyan" \ "ml-infra" "ML Infra As Service") get_or_create_endpoint $gyan_service \ "$REGION_NAME" \ "$gyan_api_url/v1" \ "$gyan_api_url/v1" \ "$gyan_api_url/v1" fi } # create_gyan_conf() - Create a new gyan.conf file function create_gyan_conf { # (Re)create ``gyan.conf`` rm -f $GYAN_CONF if [[ ${GYAN_DRIVER} == "tensorflow" ]]; then iniset $GYAN_CONF DEFAULT ml_model_driver "ml_model.driver.TensorflowDriver" fi if [[ ${GYAN_DB_TYPE} == "sql" ]]; then iniset $GYAN_CONF DEFAULT db_type sql fi iniset $GYAN_CONF DEFAULT debug "$ENABLE_DEBUG_LOG_LEVEL" iniset $GYAN_CONF DEFAULT my_ip "$HOST_IP" iniset $GYAN_CONF DEFAULT host "$HOST_IP" iniset $GYAN_CONF DEFAULT transport_url "rabbit://stackrabbit:$RABBIT_PASSWORD@$SERVICE_HOST/" iniset $GYAN_CONF oslo_messaging_rabbit rabbit_userid $RABBIT_USERID iniset $GYAN_CONF oslo_messaging_rabbit rabbit_password $RABBIT_PASSWORD iniset $GYAN_CONF oslo_messaging_rabbit rabbit_host $RABBIT_HOST iniset $GYAN_CONF database connection `database_connection_url gyan` iniset $GYAN_CONF api host_ip "$GYAN_SERVICE_HOST" iniset $GYAN_CONF api port "$GYAN_SERVICE_PORT" iniset $GYAN_CONF keystone_auth auth_type password iniset $GYAN_CONF keystone_auth username gyan iniset $GYAN_CONF keystone_auth password $SERVICE_PASSWORD iniset $GYAN_CONF keystone_auth project_name $SERVICE_PROJECT_NAME iniset $GYAN_CONF keystone_auth project_domain_id default iniset $GYAN_CONF keystone_auth user_domain_id default # FIXME(pauloewerton): keystone_authtoken section is deprecated. Remove it # after deprecation period. iniset $GYAN_CONF keystone_authtoken admin_user gyan iniset $GYAN_CONF keystone_authtoken admin_password $SERVICE_PASSWORD iniset $GYAN_CONF keystone_authtoken admin_tenant_name $SERVICE_PROJECT_NAME configure_auth_token_middleware $GYAN_CONF gyan $GYAN_AUTH_CACHE_DIR iniset $GYAN_CONF keystone_auth auth_url $KEYSTONE_AUTH_URI_V3 iniset $GYAN_CONF keystone_authtoken www_authenticate_uri $KEYSTONE_SERVICE_URI_V3 iniset $GYAN_CONF keystone_authtoken auth_url $KEYSTONE_AUTH_URI_V3 iniset $GYAN_CONF keystone_authtoken auth_version v3 iniset $GYAN_CONF compute host $HOST_IP if is_fedora || is_suse; then # gyan defaults to /usr/local/bin, but fedora and suse pip like to # install things in /usr/bin iniset $GYAN_CONF DEFAULT bindir "/usr/bin" fi if [ -n "$GYAN_STATE_PATH" ]; then iniset $GYAN_CONF DEFAULT state_path "$GYAN_STATE_PATH" iniset $GYAN_CONF oslo_concurrency lock_path "$GYAN_STATE_PATH" fi if [ "$SYSLOG" != "False" ]; then iniset $GYAN_CONF DEFAULT use_syslog "True" fi # Format logging if [ "$LOG_COLOR" == "True" ] && [ "$SYSLOG" == "False" ]; then setup_colorized_logging $GYAN_CONF DEFAULT else # Show user_name and project_name instead of user_id and project_id iniset $GYAN_CONF DEFAULT logging_context_format_string "%(asctime)s.%(msecs)03d %(levelname)s %(name)s [%(request_id)s %(user_name)s %(project_name)s] %(instance)s%(message)s" fi # Register SSL certificates if provided if is_ssl_enabled_service gyan; then ensure_certificates gyan iniset $GYAN_CONF DEFAULT ssl_cert_file "$GYAN_SSL_CERT" iniset $GYAN_CONF DEFAULT ssl_key_file "$GYAN_SSL_KEY" iniset $GYAN_CONF DEFAULT enabled_ssl_apis "$GYAN_ENABLED_APIS" fi } function create_api_paste_conf { # copy api_paste.ini cp $GYAN_DIR/etc/gyan/api-paste.ini $GYAN_API_PASTE } # create_gyan_cache_dir() - Part of the init_GYAN() process function create_gyan_cache_dir { # Create cache dir sudo mkdir -p $GYAN_AUTH_CACHE_DIR sudo chown $STACK_USER $GYAN_AUTH_CACHE_DIR rm -f $GYAN_AUTH_CACHE_DIR/* } # init_gyan() - Initialize databases, etc. function init_gyan { # Only do this step once on the API node for an entire cluster. if is_service_enabled gyan-api; then if is_service_enabled $DATABASE_BACKENDS; then # (Re)create gyan database recreate_database gyan # Migrate gyan database $GYAN_BIN_DIR/gyan-db-manage upgrade fi if is_service_enabled gyan-etcd; then install_etcd_server fi create_gyan_cache_dir fi } # install_gyanclient() - Collect source and prepare function install_gyanclient { if use_library_from_git "python-gyanclient"; then git_clone_by_name "python-gyanclient" setup_dev_lib "python-gyanclient" sudo install -D -m 0644 -o $STACK_USER {${GITDIR["python-gyanclient"]}/tools/,/etc/bash_completion.d/}gyan.bash_completion fi } # install_gyan() - Collect source and prepare function install_gyan { git_clone $GYAN_REPO $GYAN_DIR $GYAN_BRANCH setup_develop $GYAN_DIR } # start_gyan_api() - Start the API process ahead of other things function start_gyan_api { # Get right service port for testing local service_port=$GYAN_SERVICE_PORT local service_protocol=$GYAN_SERVICE_PROTOCOL if is_service_enabled tls-proxy; then service_port=$GYAN_SERVICE_PORT_INT service_protocol="http" fi local gyan_url if [ "$GYAN_USE_UWSGI" == "True" ]; then run_process gyan-api "$GYAN_BIN_DIR/uwsgi --procname-prefix gyan-api --ini $GYAN_UWSGI_CONF" gyan_url=$service_protocol://$GYAN_SERVICE_HOST/ml-infra else run_process gyan-api "$GYAN_BIN_DIR/gyan-api" gyan_url=$service_protocol://$GYAN_SERVICE_HOST:$service_port fi echo "Waiting for gyan-api to start..." if ! wait_for_service $SERVICE_TIMEOUT $gyan_url; then die $LINENO "gyan-api did not start" fi # Start proxies if enabled if is_service_enabled tls-proxy; then start_tls_proxy '*' $GYAN_SERVICE_PORT $GYAN_SERVICE_HOST $GYAN_SERVICE_PORT_INT & fi } # start_gyan_compute() - Start Gyan compute agent function start_gyan_compute { echo "Start gyan compute..." run_process gyan-compute "$GYAN_BIN_DIR/gyan-compute" } # start_gyan() - Start running processes, including screen function start_gyan { # ``run_process`` checks ``is_service_enabled``, it is not needed here mkdir -p /opt/stack/data/gyan if is_service_enabled gyan-api; then start_gyan_api fi if is_service_enabled gyan-compute; then start_gyan_compute fi } # stop_gyan() - Stop running processes (non-screen) function stop_gyan { if [ "$GYAN_USE_UWSGI" == "True" ]; then disable_apache_site gyan restart_apache_server else stop_process gyan-api fi stop_process gyan-compute } # Restore xtrace $XTRACE