Install Redis 3.2.6 by compilation

The version of Redis installed by the current elements
uses a ppa that is stalled at 3.0.7.  This seems to
have some issues that are fixed in later versions.

The recommended method of installing Redis is to compile
the code yourself, so the elements have been changed
to do this.

The version was bumped to 3.2.6 (latest stable) however
the Redis cluster tests were not reenabled as there still
seems to be an issue with them.

New config options were added to the template; these
may need to be updated to make sure all the new ones
are there.

Change-Id: I1a604b15ade815cdb51c5484d4285e504508a6c4
Partial-Bug: 1652964

integration/scripts/files/elements/ubuntu-redis/install.d/30-redis

@@ -32,9 +32,75 @@ fi
 
 _EOF_
 
-add-apt-repository -y ppa:chris-lea/redis-server
-apt-get -y update
-apt-get --allow-unauthenticated install -y redis-server
+# Install Redis from scratch
+ARCH_SAV=$ARCH
+unset ARCH
+apt-get update
+apt-get --allow-unauthenticated install -y build-essential tcl curl
+cd /tmp
+REDIS_VERSION=3.2.6
+REDIS_NAME=redis-$REDIS_VERSION
+REDIS_ARCHIVE=$REDIS_NAME.tar.gz
+curl -O http://download.redis.io/releases/$REDIS_ARCHIVE
+tar xzvf $REDIS_ARCHIVE
+cd $REDIS_NAME
+make distclean
+make
+make install
+export ARCH=$ARCH_SAV
+
+adduser --system --group --no-create-home redis
+# Create the data dir
+REDIS_DATA_DIR=/var/lib/redis
+REDIS_LOG_DIR=/var/log/redis
+REDIS_LOG=$REDIS_LOG_DIR/redis.log
+REDIS_RUN_DIR=/var/run/redis
+REDIS_PID=$REDIS_RUN_DIR/redis-server.pid
+mkdir -p $REDIS_DATA_DIR
+chown redis:redis $REDIS_DATA_DIR
+chmod 770 $REDIS_DATA_DIR
+mkdir -p $REDIS_LOG_DIR
+chown redis:redis $REDIS_LOG_DIR
+chmod 775 $REDIS_LOG_DIR
+mkdir -p $REDIS_RUN_DIR
+chown redis:redis $REDIS_RUN_DIR
+# Set up a proper conf file to start
+REDIS_CONF_NAME=redis.conf
+REDIS_CONF_DIR=/etc/redis
+REDIS_CONF=$REDIS_CONF_DIR/$REDIS_CONF_NAME
+mkdir $REDIS_CONF_DIR
+cp /tmp/$REDIS_NAME/$REDIS_CONF_NAME $REDIS_CONF_DIR
+chown redis:redis $REDIS_CONF
+sed -i "s#dir .*#dir $REDIS_DATA_DIR#" $REDIS_CONF
+sed -i "s#pidfile .*#pidfile $REDIS_PID#" $REDIS_CONF
+sed -i "s#logfile .*#logfile $REDIS_LOG#" $REDIS_CONF
+sed -i "s/supervised no/supervised systemd/" $REDIS_CONF
+sed -i "s/daemonize no/daemonize yes/" $REDIS_CONF
+sed -i "s/protected-mode yes/protected-mode no/" $REDIS_CONF
+
+cat > "/etc/systemd/system/redis-server.service" << _EOF_
+[Unit]
+Description=Redis In-Memory Data Store
+After=network.target
+
+[Service]
+Type=forking
+PIDFile=$REDIS_PID
+User=redis
+Group=redis
+
+Environment=statedir=$REDIS_RUN_DIR
+PermissionsStartOnly=true
+ExecStartPre=/bin/mkdir -p \${statedir}
+ExecStartPre=/bin/chown -R redis:redis \${statedir}
+ExecStart=/usr/local/bin/redis-server $REDIS_CONF
+ExecReload=/bin/kill -USR2 \$MAINPID
+ExecStop=/usr/local/bin/redis-cli shutdown
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
+_EOF_
 
 cat > "/etc/default/redis-server" << _EOF_
 # Call ulimit -n with this argument prior to invoking Redis itself.
@@ -46,8 +112,3 @@ _EOF_
 
 # Install Python driver for Redis ('redis-py').
 pip2 install redis
-
-# By default, redis-py will attempt to use the HiredisParser if installed.
-# Using Hiredis can provide up to a 10x speed improvement in parsing responses
-# from the Redis server.
-pip2 install hiredis

integration/scripts/trovestack

@@ -566,8 +566,8 @@ function cmd_set_datastore() {
         PACKAGES=${PACKAGES:-"mongodb-org"}
         VERSION="3.2"
     elif [ "$DATASTORE_TYPE" == "redis" ]; then
-        PACKAGES=${PACKAGES:-"redis-server"}
-        VERSION="3.0"
+        PACKAGES=${PACKAGES:-""}
+        VERSION="3.2.6"
     elif [ "$DATASTORE_TYPE" == "cassandra" ]; then
         PACKAGES=${PACKAGES:-"cassandra"}
         VERSION="2.1.0"

trove/guestagent/datastore/experimental/redis/service.py

@@ -215,6 +215,8 @@ class RedisApp(object):
 
         self.configuration_manager.apply_system_override(
             {'daemonize': 'yes',
+             'protected-mode': 'no',
+             'supervised': 'systemd',
              'pidfile': system.REDIS_PID_FILE,
              'logfile': system.REDIS_LOG_FILE,
              'dir': system.REDIS_DATA_DIR})
@@ -341,7 +343,7 @@ class RedisApp(object):
             slots = map(str, range(first_slot, last_slot + 1))
             group_size = 200
             while slots:
-                cmd = ([system.REDIS_CLI, 'cluster', 'addslots']
+                cmd = (['redis-cli', 'cluster', 'addslots']
                        + slots[0:group_size])
                 out, err = utils.execute_with_timeout(*cmd, run_as_root=True,
                                                       root_helper='sudo')

trove/guestagent/datastore/experimental/redis/system.py

@@ -28,9 +28,7 @@ REDIS_CONF_DIR = '/etc/redis'
 REDIS_DATA_DIR = '/var/lib/redis'
 REDIS_PORT = '6379'
 REDIS_INIT = '/etc/init/redis-server.conf'
-REDIS_CLI = '/usr/bin/redis-cli'
-REDIS_BIN = '/usr/bin/redis-server'
-REDIS_PACKAGE = 'redis-server'
+REDIS_PACKAGE = ''
 SERVICE_CANDIDATES = ['redis-server', 'redis']
 
 OS = get_os()

trove/templates/redis/config.template

@@ -58,6 +58,25 @@ port 6379
 # in order to get the desired effect.
 tcp-backlog 511
 
+# Protected mode is a layer of security protection, in order to avoid that
+# Redis instances left open on the internet are accessed and exploited.
+#
+# When protected mode is on and if:
+#
+# 1) The server is not binding explicitly to a set of addresses using the
+#    "bind" directive.
+# 2) No password is configured.
+#
+# The server only accepts connections from clients connecting from the
+# IPv4 and IPv6 loopback addresses 127.0.0.1 and ::1, and from Unix domain
+# sockets.
+#
+# By default protected mode is enabled. You should disable it only if
+# you are sure you want clients from other hosts to connect to Redis
+# even if no authentication is configured, nor a specific set of interfaces
+# are explicitly listed using the "bind" directive.
+protected-mode no
+
 # By default Redis listens for connections from all the network interfaces
 # available on the server. It is possible to listen to just one or multiple
 # interfaces using the "bind" configuration directive, followed by one or
@@ -96,6 +115,17 @@ timeout 0
 # A reasonable value for this option is 60 seconds.
 tcp-keepalive 0
 
+# If you run Redis from upstart or systemd, Redis can interact with your
+# supervision tree. Options:
+#   supervised no      - no supervision interaction
+#   supervised upstart - signal upstart by putting Redis into SIGSTOP mode
+#   supervised systemd - signal systemd by writing READY=1 to $NOTIFY_SOCKET
+#   supervised auto    - detect upstart or systemd method based on
+#                        UPSTART_JOB or NOTIFY_SOCKET environment variables
+# Note: these supervision methods only signal "process is ready."
+#       They do not enable continuous liveness pings back to your supervisor.
+supervised systemd
+
 # Specify the server verbosity level.
 # This can be one of:
 # debug (a lot of information, useful for development/testing)
@@ -844,6 +874,37 @@ hash-max-ziplist-value 64
 list-max-ziplist-entries 512
 list-max-ziplist-value 64
 
+# Lists are also encoded in a special way to save a lot of space.
+# The number of entries allowed per internal list node can be specified
+# as a fixed maximum size or a maximum number of elements.
+# For a fixed maximum size, use -5 through -1, meaning:
+# -5: max size: 64 Kb  <-- not recommended for normal workloads
+# -4: max size: 32 Kb  <-- not recommended
+# -3: max size: 16 Kb  <-- probably not recommended
+# -2: max size: 8 Kb   <-- good
+# -1: max size: 4 Kb   <-- good
+# Positive numbers mean store up to _exactly_ that number of elements
+# per list node.
+# The highest performing option is usually -2 (8 Kb size) or -1 (4 Kb size),
+# but if your use case is unique, adjust the settings as necessary.
+list-max-ziplist-size -2
+#
+# Lists may also be compressed.
+# Compress depth is the number of quicklist ziplist nodes from *each* side of
+# the list to *exclude* from compression.  The head and tail of the list
+# are always uncompressed for fast push/pop operations.  Settings are:
+# 0: disable all list compression
+# 1: depth 1 means "don't start compressing until after 1 node into the list,
+#    going from either the head or tail"
+#    So: [head]->node->node->...->node->[tail]
+#    [head], [tail] will always be uncompressed; inner nodes will compress.
+# 2: [head]->[next]->node->node->...->node->[prev]->[tail]
+#    2 here means: don't compress head or head->next or tail->prev or tail,
+#    but compress all nodes between them.
+# 3: [head]->[next]->[next]->node->node->...->node->[prev]->[prev]->[tail]
+# etc.
+list-compress-depth 0
+
 # Sets have a special encoding in just one case: when a set is composed
 # of just strings that happen to be integers in radix 10 in the range
 # of 64 bit signed integers.
@@ -951,4 +1012,4 @@ hz 10
 # the file will be fsync-ed every 32 MB of data generated. This is useful
 # in order to commit the file to the disk more incrementally and avoid
 # big latency spikes.
-aof-rewrite-incremental-fsync yes
+aof-rewrite-incremental-fsync yes

trove/tests/int_tests.py

@@ -405,10 +405,10 @@ register(
     single=[common_groups,
             backup_groups,
             configuration_groups, ],
-    multi=[
-        # cluster_actions_groups,
-        # cluster_negative_actions_groups,
-        replication_promote_groups, ]
+    multi=[replication_promote_groups, ]
+    # multi=[cluster_actions_groups,
+    #        cluster_negative_actions_groups,
+    #        replication_promote_groups, ]
 )
 
 register(

trove/tests/scenario/helpers/test_helper.py

@@ -39,18 +39,18 @@ class DataType(Enum):
     micro4 = 4
     # very tiny amount of data, useful for testing replication
     # propagation, etc.
-    tiny = 3
+    tiny = 5
     # another tiny dataset (also for replication propagation)
-    tiny2 = 4
+    tiny2 = 6
     # a third tiny dataset (also for replication propagation)
-    tiny3 = 5
+    tiny3 = 7
     # a forth tiny dataset (for cluster propagation)
-    tiny4 = 6
+    tiny4 = 8
     # small amount of data (this can be added to each instance
     # after creation, for example).
-    small = 7
+    small = 9
     # large data, enough to make creating a backup take 20s or more.
-    large = 8
+    large = 10
 
 
 class TestHelper(object):