Enable graceful shutdown for services

1] Process using uwsgi:
   uwsgi services doesn't support for graceful shutting down [1].
   It requires some changes in unit files [2] including adding below
   graceful shutdown hook and changing KillSignal:
      --hook-master-start "unix_signal:15 gracefully_kill_them_all
   All the steps and changes required are specified in etherpad [1].

2] Non uwsgi services needs below changes:
   In [service] section:
   a. Add KillMode = process
   b. Add TimeoutStopSec = infinity

NOTE:
Creating unit file for services other than uwsgi is handled by the
'write_user_unit_file' function [3]. This function is common for all
the services so this patch adds the above mentioned parameters for
services using ServiceLauncher also though they don't require.

Added a new stackrc variable WORKER_TIMEOUT which is required to add
graceful shutdown support to uwsgi services. It will be set as a value
to 'worker-reload-mercy' [4] in uwsgi file of service. The default
value set to this variable is 90.

[1] https://etherpad.openstack.org/p/uwsgi-issues
[2] https://www.freedesktop.org/software/systemd/man/systemd.kill.html
[3] 2967ca3dfd/functions-common (L1439-L1461)
[4] http://uwsgi-docs.readthedocs.io/en/latest/Options.html#worker-reload-mercy

Co-Authored-By: Dinesh Bhor <dinesh.bhor@nttdata.com>
Change-Id: Ia95291325ce4858b47102dd49504250183f339ab

functions-common

@@ -1451,6 +1451,8 @@ function write_user_unit_file {
     iniset -sudo $unitfile "Unit" "Description" "Devstack $service"
     iniset -sudo $unitfile "Service" "User" "$user"
     iniset -sudo $unitfile "Service" "ExecStart" "$command"
+    iniset -sudo $unitfile "Service" "KillMode" "process"
+    iniset -sudo $unitfile "Service" "TimeoutStopSec" "infinity"
     if [[ -n "$group" ]]; then
         iniset -sudo $unitfile "Service" "Group" "$group"
     fi
@@ -1473,7 +1475,7 @@ function write_uwsgi_user_unit_file {
     iniset -sudo $unitfile "Service" "User" "$user"
     iniset -sudo $unitfile "Service" "ExecStart" "$command"
     iniset -sudo $unitfile "Service" "Type" "notify"
-    iniset -sudo $unitfile "Service" "KillSignal" "SIGQUIT"
+    iniset -sudo $unitfile "Service" "KillMode" "process"
     iniset -sudo $unitfile "Service" "Restart" "always"
     iniset -sudo $unitfile "Service" "NotifyAccess" "all"
     iniset -sudo $unitfile "Service" "RestartForceExitStatus" "100"

lib/apache

@@ -260,10 +260,15 @@ function write_uwsgi_config {
     # Set die-on-term & exit-on-reload so that uwsgi shuts down
     iniset "$file" uwsgi die-on-term true
     iniset "$file" uwsgi exit-on-reload true
+    # Set worker-reload-mercy so that worker will not exit till the time
+    # configured after graceful shutdown
+    iniset "$file" uwsgi worker-reload-mercy $WORKER_TIMEOUT
     iniset "$file" uwsgi enable-threads true
     iniset "$file" uwsgi plugins python
     # uwsgi recommends this to prevent thundering herd on accept.
     iniset "$file" uwsgi thunder-lock true
+    # Set hook to trigger graceful shutdown on SIGTERM
+    iniset "$file" uwsgi hook-master-start "unix_signal:15 gracefully_kill_them_all"
     # Override the default size for headers from the 4k default.
     iniset "$file" uwsgi buffer-size 65535
     # Make sure the client doesn't try to re-use the connection.
@@ -316,6 +321,11 @@ function write_local_uwsgi_http_config {
     iniset "$file" uwsgi plugins python
     # uwsgi recommends this to prevent thundering herd on accept.
     iniset "$file" uwsgi thunder-lock true
+    # Set hook to trigger graceful shutdown on SIGTERM
+    iniset "$file" uwsgi hook-master-start "unix_signal:15 gracefully_kill_them_all"
+    # Set worker-reload-mercy so that worker will not exit till the time
+    # configured after graceful shutdown
+    iniset "$file" uwsgi worker-reload-mercy $WORKER_TIMEOUT
     # Override the default size for headers from the 4k default.
     iniset "$file" uwsgi buffer-size 65535
     # Make sure the client doesn't try to re-use the connection.

stackrc

@@ -778,6 +778,9 @@ SERVICE_TIMEOUT=${SERVICE_TIMEOUT:-60}
 # Service graceful shutdown timeout
 SERVICE_GRACEFUL_SHUTDOWN_TIMEOUT=${SERVICE_GRACEFUL_SHUTDOWN_TIMEOUT:-5}
 
+# Service graceful shutdown timeout
+WORKER_TIMEOUT=${WORKER_TIMEOUT:-90}
+
 # Support alternative yum -- in future Fedora 'dnf' will become the
 # only supported installer, but for now 'yum' and 'dnf' are both
 # available in parallel with compatible CLIs.  Allow manual switching