diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 676d2546c..98731bf03 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -284,8 +284,6 @@ function configure_df_plugin { neutron_server_config_add_new $DRAGONFLOW_CONF fi - iniset $DRAGONFLOW_CONF df remote_db_ip "$REMOTE_DB_IP" - iniset $DRAGONFLOW_CONF df remote_db_port $REMOTE_DB_PORT iniset $DRAGONFLOW_CONF df remote_db_hosts "$REMOTE_DB_HOSTS" iniset $DRAGONFLOW_CONF df nb_db_class "$NB_DRIVER_CLASS" iniset $DRAGONFLOW_CONF df enable_neutron_notifier "$ENABLE_NEUTRON_NOTIFIER" diff --git a/devstack/redis_driver b/devstack/redis_driver index bcc8fcb2e..aad77d861 100644 --- a/devstack/redis_driver +++ b/devstack/redis_driver @@ -15,8 +15,9 @@ RUBY_VERSION=2.3 function _redis_env { # REMOTE_DB_* initialized after sourcing - export REDIS_SERVER_LIST=$REMOTE_DB_IP - export REMOTE_PORT_START=$REMOTE_DB_PORT + export REDIS_SERVER_LIST=$(echo $REMOTE_DB_HOSTS | sed 's/,/ /g') + export REMOTE_PORT_START=$(echo $REDIS_SERVER_LIST | awk '{print $1}' | cut -d: -f2) + export REDIS_SERVER_IPS=$(echo $REDIS_SERVER_LIST | awk -F: 'BEGIN {RS=" "} { print $1 }' | sort | uniq) export NODE_COUNT_END=5 export REMOTE_PORT_END=`expr $REMOTE_PORT_START + $NODE_COUNT_END` export REDIS_PORT=`seq $REMOTE_PORT_START $REMOTE_PORT_END` @@ -33,7 +34,7 @@ function _configure_redis { sudo sh -c "grep -q vm.overcommit_memory /etc/sysctl.conf || echo vm.overcommit_memory = 1 >> /etc/sysctl.conf" sudo sh -c "sysctl -p" - for redisserver in $REDIS_SERVER_LIST; do + for redisserver in $REDIS_SERVER_IPS; do for port in $REDIS_PORT; do echo "shutdown redis: "$redisserver:$port cp redis.conf /opt/redis3/conf/redis-$port.conf @@ -118,28 +119,28 @@ function nb_db_driver_start_server { test -f /opt/redis3/conf/nodes-$port.conf || { create=true; break 2 ; } done done - # start redis - for redisserver in $REDIS_SERVER_LIST; do - for port in $REDIS_PORT; do - echo $redisserver:$port - pushd /opt/redis3/ - [ "$create" ] && { + # start redis + for redisserver in $REDIS_SERVER_IPS; do + for port in $REDIS_PORT; do + echo $redisserver:$port + pushd /opt/redis3/ + [ "$create" ] && { - sudo rm nodes* -rf - } + sudo rm nodes* -rf + } - ./bin/redis-server ./conf/redis-$port.conf & - redis_cluster="$redis_cluster"" ""$redisserver:$port" - popd - done - done - # create cluster - [ "$create" ] && { - echo "Create the Redis cluster: "$redis_cluster - pushd /opt/redis3/bin/ - echo "yes" | sudo ./redis-trib.rb create --replicas 1 $redis_cluster - popd - } + ./bin/redis-server ./conf/redis-$port.conf & + redis_cluster="$redis_cluster"" ""$redisserver:$port" + popd + done + done + # create cluster + [ "$create" ] && { + echo "Create the Redis cluster: "$redis_cluster + pushd /opt/redis3/bin/ + echo "yes" | sudo ./redis-trib.rb create --replicas 1 $redis_cluster + popd + } fi fi @@ -149,7 +150,7 @@ function nb_db_driver_stop_server { _redis_env if is_service_enabled df-redis-server ; then if is_ubuntu || is_fedora; then - for redisserver in $REDIS_SERVER_LIST; do + for redisserver in $REDIS_SERVER_IPS; do for port in $REDIS_PORT; do echo "Shutdown Redis: "$redisserver:$port sudo /opt/redis3/bin/redis-cli -p $port shutdown diff --git a/devstack/zookeeper_driver b/devstack/zookeeper_driver index 675f0b080..2e340bf28 100644 --- a/devstack/zookeeper_driver +++ b/devstack/zookeeper_driver @@ -20,6 +20,8 @@ function _zookeeper_env { export ZOOKEEPER_DATA_DIR="/var/lib/zookeeper" export ZOOKEEPER_LOG_DIR="/var/log/zookeeper" export ZOOKEEPER_DIR="/etc/zookeeper" + local SERVER_LIST=$(echo $REMOTE_DB_HOSTS | sed 's/,/ /g') + export ZOOKEEPER_SERVER_PORT=$(echo $SERVER_LIST | awk -F: 'BEGIN {RS=" "} { print $2 }' | sort | uniq | tail -1) } function update_key_in_file { @@ -58,7 +60,7 @@ function nb_db_driver_install_server { if [ -f $ZOOKEEPER_CONF_FILE ] ; then update_key_in_file dataDir "${ZOOKEEPER_DATA_DIR}" $ZOOKEEPER_CONF_FILE update_key_in_file dataLogDir "${ZOOKEEPER_LOG_DIR}" $ZOOKEEPER_CONF_FILE - update_key_in_file clientPort "${REMOTE_DB_PORT}" $ZOOKEEPER_CONF_FILE + update_key_in_file clientPort "${ZOOKEEPER_SERVER_PORT}" $ZOOKEEPER_CONF_FILE update_key_in_file "server.1" "${HOSTNAME}:2888:3888" $ZOOKEEPER_CONF_FILE fi sudo systemctl restart zookeeper diff --git a/doc/source/manual_deployment.rst b/doc/source/manual_deployment.rst index 2a8e79821..8ee078a7a 100644 --- a/doc/source/manual_deployment.rst +++ b/doc/source/manual_deployment.rst @@ -125,8 +125,7 @@ Next you need to change the configuration, for example, etcd: [df] nb_db_class = etcd_nb_db_driver - remote_db_port = {etcd_port} - remote_db_ip = {etcd_ip} + remote_db_hosts = [{etcd_ip}:{etcd_port}] Pub/Sub Driver -------------- diff --git a/doc/source/multi-node-conf/etcd_compute_node.conf b/doc/source/multi-node-conf/etcd_compute_node.conf index 32c8d881d..dbccd6737 100644 --- a/doc/source/multi-node-conf/etcd_compute_node.conf +++ b/doc/source/multi-node-conf/etcd_compute_node.conf @@ -29,7 +29,7 @@ enable_service df-metadata SERVICE_HOST= RABBIT_HOST=$SERVICE_HOST Q_HOST=$SERVICE_HOST -REMOTE_DB_IP=$SERVICE_HOST +REMOTE_DB_HOSTS="$SERVICE_HOST:2379" # Make VNC work on compute node NOVA_VNC_ENABLED=True diff --git a/doc/source/multi-node-conf/ramcloud_compute_node.conf b/doc/source/multi-node-conf/ramcloud_compute_node.conf index 68c4391a5..91df2073e 100644 --- a/doc/source/multi-node-conf/ramcloud_compute_node.conf +++ b/doc/source/multi-node-conf/ramcloud_compute_node.conf @@ -39,7 +39,7 @@ REMOTE_DB_PORT=21222 SERVICE_HOST= RABBIT_HOST=$SERVICE_HOST Q_HOST=$SERVICE_HOST -REMOTE_DB_IP=$SERVICE_HOST +REMOTE_DB_HOSTS="$SERVICE_HOST:4001" # Make VNC work on compute node NOVA_VNC_ENABLED=True diff --git a/doc/source/multi-node-conf/redis_compute_node_local_controller.conf b/doc/source/multi-node-conf/redis_compute_node_local_controller.conf index 1dd7814c1..6e9ea05e6 100644 --- a/doc/source/multi-node-conf/redis_compute_node_local_controller.conf +++ b/doc/source/multi-node-conf/redis_compute_node_local_controller.conf @@ -39,7 +39,7 @@ enable_service df-metadata SERVICE_HOST= RABBIT_HOST=$SERVICE_HOST Q_HOST=$SERVICE_HOST -REMOTE_DB_IP=$SERVICE_HOST +REMOTE_DB_HOSTS="$SERVICE_HOST:4001" # Make VNC work on compute node NOVA_VNC_ENABLED=True diff --git a/doc/source/multi-node-conf/rethinkDB_compute_node.conf b/doc/source/multi-node-conf/rethinkDB_compute_node.conf index c6e44c2a8..c2f81acb3 100644 --- a/doc/source/multi-node-conf/rethinkDB_compute_node.conf +++ b/doc/source/multi-node-conf/rethinkDB_compute_node.conf @@ -39,7 +39,7 @@ REMOTE_DB_PORT=28015 SERVICE_HOST= RABBIT_HOST=$SERVICE_HOST Q_HOST=$SERVICE_HOST -REMOTE_DB_IP=$SERVICE_HOST +REMOTE_DB_HOSTS="$SERVICE_HOST:4001" # Make VNC work on compute node NOVA_VNC_ENABLED=True diff --git a/dragonflow/conf/df_common_params.py b/dragonflow/conf/df_common_params.py index c957f6b31..df2a2e0cb 100644 --- a/dragonflow/conf/df_common_params.py +++ b/dragonflow/conf/df_common_params.py @@ -18,9 +18,11 @@ from dragonflow._i18n import _ df_opts = [ cfg.IPOpt('remote_db_ip', default='127.0.0.1', + deprecated_for_removal=True, help=_('The remote db server ip address')), cfg.PortOpt('remote_db_port', default=4001, + deprecated_for_removal=True, help=_('The remote db server port')), cfg.ListOpt('remote_db_hosts', default=['$remote_db_ip:$remote_db_port'], diff --git a/dragonflow/db/api_nb.py b/dragonflow/db/api_nb.py index da925e156..2216e7c21 100644 --- a/dragonflow/db/api_nb.py +++ b/dragonflow/db/api_nb.py @@ -34,6 +34,19 @@ LOG = log.getLogger(__name__) _nb_api = None +def get_db_ip_port(): + hosts = cfg.CONF.df.remote_db_hosts + if not hosts: + LOG.warning("Deprecated: remote_db_ip and remote_db_port are " + "deprecated for removal. Use remote_db_hosts instead") + ip = cfg.CONF.df.remote_db_ip + port = cfg.CONF.df.remote_db_port + return ip, port + host = hosts[0] + ip, port = host.split(':') + return ip, port + + def _get_topic(obj): try: return getattr(obj, 'topic', None) @@ -70,8 +83,8 @@ class NbApi(object): nb_driver, use_pubsub=cfg.CONF.df.enable_df_pub_sub, is_neutron_server=is_neutron_server) - nb_api.initialize(db_ip=cfg.CONF.df.remote_db_ip, - db_port=cfg.CONF.df.remote_db_port) + ip, port = get_db_ip_port() + nb_api.initialize(db_ip=ip, db_port=port) _nb_api = nb_api return _nb_api diff --git a/dragonflow/db/pubsub_drivers/etcd_pubsub_driver.py b/dragonflow/db/pubsub_drivers/etcd_pubsub_driver.py index 4e955fc92..5603741d2 100644 --- a/dragonflow/db/pubsub_drivers/etcd_pubsub_driver.py +++ b/dragonflow/db/pubsub_drivers/etcd_pubsub_driver.py @@ -15,9 +15,9 @@ import etcd3gw import threading -from oslo_config import cfg from oslo_log import log as logging +from dragonflow.db import api_nb from dragonflow.db import pub_sub_api @@ -51,8 +51,8 @@ class EtcdPublisherAgent(pub_sub_api.PublisherAgentBase): def initialize(self): super(EtcdPublisherAgent, self).__init__() - self.client = etcd3gw.client(host=cfg.CONF.df.remote_db_ip, - port=cfg.CONF.df.remote_db_port) + ip, port = api_nb.get_db_ip_port() + self.client = etcd3gw.client(host=ip, port=port) def _send_event(self, data, topic): topic_prefix = _get_topic_watch_prefix(topic) @@ -94,8 +94,8 @@ class EtcdSubscriberAgent(pub_sub_api.SubscriberApi): def initialize(self, callback): self.db_changes_callback = callback self.stop_event = threading.Event() - self.client = etcd3gw.client(host=cfg.CONF.df.remote_db_ip, - port=cfg.CONF.df.remote_db_port) + ip, port = api_nb.get_db_ip_port() + self.client = etcd3gw.client(host=ip, port=port) def _create_topic_thread(self, topic): topic_thread = WatcherThread( diff --git a/dragonflow/db/pubsub_drivers/redis_db_pubsub_driver.py b/dragonflow/db/pubsub_drivers/redis_db_pubsub_driver.py index 2b1aa3fd6..3542ff5c1 100644 --- a/dragonflow/db/pubsub_drivers/redis_db_pubsub_driver.py +++ b/dragonflow/db/pubsub_drivers/redis_db_pubsub_driver.py @@ -18,8 +18,8 @@ from oslo_log import log as logging from oslo_serialization import jsonutils import redis -from dragonflow import conf as cfg from dragonflow.controller.common import constants +from dragonflow.db import api_nb from dragonflow.db.drivers import redis_mgt from dragonflow.db import pub_sub_api @@ -54,9 +54,8 @@ class RedisPublisherAgent(pub_sub_api.PublisherAgentBase): def initialize(self): # find a publisher server node super(RedisPublisherAgent, self).initialize() - self.redis_mgt = redis_mgt.RedisMgt.get_instance( - cfg.CONF.df.remote_db_ip, - cfg.CONF.df.remote_db_port) + ip, port = api_nb.get_db_ip_port() + self.redis_mgt = redis_mgt.RedisMgt.get_instance(ip, port) self._update_client() def close(self): @@ -136,9 +135,8 @@ class RedisSubscriberAgent(pub_sub_api.SubscriberAgentBase): def initialize(self, callback): # find a subscriber server node and run daemon super(RedisSubscriberAgent, self).initialize(callback) - self.redis_mgt = redis_mgt.RedisMgt.get_instance( - cfg.CONF.df.remote_db_ip, - cfg.CONF.df.remote_db_port) + ip, port = api_nb.get_db_ip_port() + self.redis_mgt = redis_mgt.RedisMgt.get_instance(ip, port) self._update_client() self.is_closed = False diff --git a/dragonflow/tests/fullstack/test_db_api.py b/dragonflow/tests/fullstack/test_db_api.py index e97749300..42bb49267 100644 --- a/dragonflow/tests/fullstack/test_db_api.py +++ b/dragonflow/tests/fullstack/test_db_api.py @@ -12,6 +12,7 @@ from dragonflow.common import utils as df_utils from dragonflow import conf as cfg +from dragonflow.db import api_nb from dragonflow.tests.database import test_db_api from dragonflow.tests.fullstack import test_base @@ -23,6 +24,5 @@ class TestDbApi(test_base.DFTestBase, test_db_api.TestDbApi): self.driver = df_utils.load_driver( cfg.CONF.df.nb_db_class, df_utils.DF_NB_DB_DRIVER_NAMESPACE) - self.driver.initialize(cfg.CONF.df.remote_db_ip, - cfg.CONF.df.remote_db_port, - config=cfg.CONF.df) + ip, port = api_nb.get_db_ip_port() + self.driver.initialize(ip, port, config=cfg.CONF.df)