WIP: Ensure systemd services are activated

When deployment times out during running the paunch command, containers
might be left in an incomplete status. Especially when podman is used
and containers are managed by systemd, sometimes paunch succeeds to
create a container but it is aborted before creating and enabling
the systemd service for that container.

This change makes paunch check status of systemd services, to ensure
any disabled/stopped/missing services are properly recovered by next
run.

Closes-Bug: #1955056
Change-Id: I1713d55318834cd93a0a045970aa4800aed4cd68

paunch/builder/base.py

@@ -102,6 +102,43 @@ class BaseBuilder(object):
                 if container in desired_names:
                     self.log.debug('Skipping existing container: %s' %
                                    container)
+
+                    if systemd_managed:
+                        try:
+                            if not systemd.service_enabled(
+                                    container=container_name, log=self.log):
+                                self.log.warning(
+                                    'Service for container %s is not '
+                                    'activated. Reactivating' % container_name)
+                                systemd.service_create(
+                                    container=container_name,
+                                    cconfig=cconfig,
+                                    log=self.log)
+
+                            if (not self.healthcheck_disabled and
+                                    'healthcheck' in cconfig and
+                                    not systemd.healthcheck_enabled(
+                                        container=container_name,
+                                        log=self.log)):
+                                self.log.warning(
+                                    'Healthcheck service for container %s is '
+                                    'not activated. Reactivating'
+                                    % container_name)
+                                check = cconfig.get('healthcheck')['test']
+                                systemd.healthcheck_create(
+                                    container=container_name,
+                                    log=self.log,
+                                    test=check)
+                                systemd.healthcheck_timer_create(
+                                    container=container_name,
+                                    cconfig=cconfig,
+                                    log=self.log)
+                        except systemctl.SystemctlMaskedException:
+                            self.log.warning(
+                                'Masked service for container %s '
+                                'is not managed here' % container_name)
+                            pass
+
                     continue
 
                 c_name = self.runner.discover_container_name(

paunch/tests/test_utils_systemctl.py

@@ -60,6 +60,15 @@ class TestUtilsSystemctl(base.TestCase):
                       stderr=-1, stdout=-1, universal_newlines=True)
         ])
 
+    @mock.patch('subprocess.run', autospec=True)
+    def test_is_enabled(self, mock_subprocess_run):
+        mock_subprocess_run.return_value = self.r
+        systemctl.is_active('foo')
+        mock_subprocess_run.assert_has_calls([
+            mock.call(['systemctl', 'is-enabled', '-q', 'foo'],
+                      stderr=-1, stdout=-1, universal_newlines=True)
+        ])
+
     @mock.patch('subprocess.run', autospec=True)
     def test_enable(self, mock_subprocess_run):
         mock_subprocess_run.return_value = self.r

paunch/tests/test_utils_systemd.py

@@ -131,6 +131,48 @@ class TestUtilsSystemd(base.TestCase):
             mock.call(os.path.join(tempdir, service_requires_d)),
         ])
 
+    @mock.patch('os.path.isfile', autospec=True)
+    @mock.patch('subprocess.run', autospec=True)
+    def test_service_is_active(self, mock_subprocess_run, mock_isfile):
+        mock_subprocess_run.return_value = self.r
+        mock_isfile.return_value = True
+        container = 'my_app'
+        service = 'tripleo_' + container
+
+        self.assertTrue(systemd.service_is_active(container))
+        mock_subprocess_run.assert_has_calls([
+            mock.call(['systemctl', 'is-enabled', '-q', service + '.service'],
+                      stderr=-1, stdout=-1, universal_newlines=True),
+            mock.call(['systemctl', 'is-active', '-q', service + '.service'],
+                      stderr=-1, stdout=-1, universal_newlines=True),
+        ])
+
+    @mock.patch('os.path.isfile', autospec=True)
+    @mock.patch('subprocess.run', autospec=True)
+    def test_service_is_active_file_not_exist(self, mock_subprocess_run,
+                                              mock_isfile):
+        mock_subprocess_run.return_value = self.r
+        mock_isfile.return_value = False
+        container = 'my_app'
+
+        self.assertFalse(systemd.service_is_active(container))
+        self.assertEqual(0, mock_subprocess_run.call_count)
+
+    @mock.patch('os.path.isfile', autospec=True)
+    @mock.patch('subprocess.run', autospec=True)
+    def test_service_is_active_stopped(self, mock_subprocess_run, mock_isfile):
+        mock_subprocess_run.return_value = self.r
+        self.r.return_code = 1
+        mock_isfile.return_value = True
+        container = 'my_app'
+        service = 'tripleo_' + container
+
+        self.assertFalse(systemd.service_is_active(container))
+        mock_subprocess_run.assert_has_calls([
+            mock.call(['systemctl', 'is-enabled', '-q', service + '.service'],
+                      stderr=-1, stdout=-1, universal_newlines=True)
+        ])
+
     @mock.patch('os.chmod')
     def test_healthcheck_create(self, mock_chmod):
         container = 'my_app'

paunch/utils/systemctl.py

@@ -61,6 +61,10 @@ def is_active(service, log=None):
     systemctl(['is-active', '-q', service], log)
 
 
+def is_enabled(service, log=None):
+    systemctl(['is-enabled', '-q', service], log)
+
+
 def is_masked(service, log=None):
     out = systemctl(['is-enabled', service], log, ignore_errors=True)
     return 'masked' in out

paunch/utils/systemd.py

@@ -159,6 +159,25 @@ def service_delete(container, sysdir=constants.SYSTEMD_DIR, log=None):
             shutil.rmtree(os.path.join(sysdir, sysd_health_req_d))
 
 
+def service_is_active(container, log=None):
+    log = log or common.configure_logging(__name__)
+    # prefix is explained in the service_create().
+    service = 'tripleo_' + container
+
+    sysd_unit_f = systemctl.format_name(service)
+    if not os.path.isfile(sysdir + sysd_unit_f):
+        return False
+
+    try:
+        systemctl.is_enabled(sysd_f)
+        systemctl.is_active(sysd_f)
+    except systemctl.SystemctlException:
+        # The service is stopped, disabled or not registered
+        return False
+
+    return True
+
+
 def healthcheck_create(container, sysdir='/etc/systemd/system/',
                        log=None, test='/openstack/healthcheck'):
     """Create a healthcheck for a service in systemd
@@ -252,3 +271,24 @@ WantedBy=timers.target""" % s_config)
     except systemctl.SystemctlException:
         log.exception("systemctl failed")
         raise
+
+
+def healthcheck_is_active(container, log=None):
+    log = log or
+    log = log or common.configure_logging(__name__)
+
+    service = 'tripleo_' + container
+    healthcheck_timer = service + '_healthcheck.timer'
+    sysd_timer_f = sysdir + healthcheck_timer
+
+    if not os.path.isfile(sysdir + sysd_timer_f):
+        return False
+
+    try:
+        systemctl.is_enabled(sysd_timer_f)
+        systemctl.is_active(sysd_timer_f)
+    except systemctl.SystemctlException:
+        # The service is stopped, disabled or not registered
+        return False
+
+    return True