diff --git a/kuryr_tempest_plugin/tests/scenario/base.py b/kuryr_tempest_plugin/tests/scenario/base.py
index 21aea073..58f670c3 100644
--- a/kuryr_tempest_plugin/tests/scenario/base.py
+++ b/kuryr_tempest_plugin/tests/scenario/base.py
@@ -747,9 +747,13 @@ class BaseKuryrScenarioTest(manager.NetworkScenarioTest):
             namespace=namespace, label_selector=label_selector)]
 
     def get_controller_pod_names(self):
-        return self.get_pod_name_list(
+        controller_label = CONF.kuryr_kubernetes.controller_label
+        controller_pod_names = self.get_pod_name_list(
             namespace=CONF.kuryr_kubernetes.kube_system_namespace,
-            label_selector=CONF.kuryr_kubernetes.controller_label)
+            label_selector=controller_label)
+        self.assertNotEmpty(controller_pod_names, "Can't find controller pods "
+                            "with label %s" % controller_label)
+        return controller_pod_names
 
     def _run_and_assert_fn(self, fn, repeats=10, responses_num=2):
         cmd_outputs = set()
diff --git a/kuryr_tempest_plugin/tests/scenario/test_service.py b/kuryr_tempest_plugin/tests/scenario/test_service.py
index b0c6dea9..93f5077d 100644
--- a/kuryr_tempest_plugin/tests/scenario/test_service.py
+++ b/kuryr_tempest_plugin/tests/scenario/test_service.py
@@ -11,6 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import testtools
+
 from oslo_log import log as logging
 from tempest import config
 from tempest.lib import decorators
@@ -83,6 +85,9 @@ class TestLoadBalancerServiceScenario(base.BaseKuryrScenarioTest):
         self._run_and_assert_fn(curl)
 
     @decorators.idempotent_id('bddf5441-1244-449d-a125-b5fdbfc1b2a7')
+    @testtools.skipUnless(
+        CONF.kuryr_kubernetes.containerized,
+        "test_unsupported_service_type only runs on containerized setups")
     def test_unsupported_service_type(self):
         # Testing that kuryr controller didn't crash for 100 seconds since
         # creation of service with unsupported type