diff --git a/nova/compute/rpcapi.py b/nova/compute/rpcapi.py
index bd451fd450a9..d3d4a57f9a6f 100644
--- a/nova/compute/rpcapi.py
+++ b/nova/compute/rpcapi.py
@@ -341,6 +341,17 @@ class ComputeAPI(object):
             return LAST_VERSION
         service_version = objects.Service.get_minimum_version(
             context.get_admin_context(), 'nova-compute')
+
+        # NOTE(johngarbutt) when there are no nova-compute services running we
+        # get service_version == 0. In that case we do not want to cache
+        # this result, because we will get a better answer next time.
+        # As a sane default, return the version from the last release.
+        if service_version == 0:
+            LOG.debug("Not caching compute RPC version_cap, because min "
+                      "service_version is 0. Please ensure a nova-compute "
+                      "service has been started. Defaulting to Mitaka RPC.")
+            return self.VERSION_ALIASES["mitaka"]
+
         history = service_obj.SERVICE_VERSION_HISTORY
         try:
             version_cap = history[service_version]['compute_rpc']
diff --git a/nova/tests/unit/compute/test_rpcapi.py b/nova/tests/unit/compute/test_rpcapi.py
index da307c741ad8..bbe2ea2d453d 100644
--- a/nova/tests/unit/compute/test_rpcapi.py
+++ b/nova/tests/unit/compute/test_rpcapi.py
@@ -71,13 +71,14 @@ class ComputeRpcAPITestCase(test.NoDBTestCase):
                           compute_rpcapi.ComputeAPI)
 
     @mock.patch('nova.objects.Service.get_minimum_version')
-    def test_auto_pin_kilo(self, mock_get_min):
+    def test_auto_pin_with_service_version_zero(self, mock_get_min):
         mock_get_min.return_value = 0
         self.flags(compute='auto', group='upgrade_levels')
         compute_rpcapi.LAST_VERSION = None
         rpcapi = compute_rpcapi.ComputeAPI()
-        self.assertEqual('4.0', rpcapi.client.version_cap)
+        self.assertEqual('4.11', rpcapi.client.version_cap)
         mock_get_min.assert_called_once_with(mock.ANY, 'nova-compute')
+        self.assertIsNone(compute_rpcapi.LAST_VERSION)
 
     @mock.patch('nova.objects.Service.get_minimum_version')
     def test_auto_pin_caches(self, mock_get_min):