Ensure sane default memory configuration

Cap default memory consumption at 512MB or 50% of system memory -
whichever is the lowest - to ensure that deployments in containers
on large server infrastructure don't end up consuming silly
amounts of memory by default.

Update tests to not explicity configure dataset-size, and to
remove the constraint for a larger VM for testing - with the
performance schema disabled by default things should fit into
a default instance size in the gate.

Change-Id: Ifb9bd7e28e160190913f6bd03b84073a2857e4d7

charmhelpers/contrib/database/mysql.py

@@ -17,6 +17,7 @@ import sys
 import platform
 import os
 import glob
+import six
 
 # from string import upper
 
@@ -50,7 +51,10 @@ try:
     import MySQLdb
 except ImportError:
     apt_update(fatal=True)
-    apt_install(filter_installed_packages(['python-mysqldb']), fatal=True)
+    if six.PY2:
+        apt_install(filter_installed_packages(['python-mysqldb']), fatal=True)
+    else:
+        apt_install(filter_installed_packages(['python3-mysqldb']), fatal=True)
     import MySQLdb
 
 
@@ -330,6 +334,7 @@ class PerconaClusterHelper(object):
 
     DEFAULT_PAGE_SIZE = 16 * 1024 * 1024
     DEFAULT_INNODB_BUFFER_FACTOR = 0.50
+    DEFAULT_INNODB_BUFFER_SIZE_MAX = 512 * 1024 * 1024
 
     def human_to_bytes(self, human):
         """Convert human readable configuration options to bytes."""
@@ -412,8 +417,14 @@ class PerconaClusterHelper(object):
             innodb_buffer_pool_size = self.human_to_bytes(
                 dataset_bytes)
         else:
-            innodb_buffer_pool_size = int(
-                total_memory * self.DEFAULT_INNODB_BUFFER_FACTOR)
+            # NOTE(jamespage): pick the smallest of 50% of RAM or 512MB
+            #                  to ensure that deployments in containers
+            #                  without constraints don't try to consume
+            #                  silly amounts of memory.
+            innodb_buffer_pool_size = min(
+                int(total_memory * self.DEFAULT_INNODB_BUFFER_FACTOR),
+                self.DEFAULT_INNODB_BUFFER_SIZE_MAX
+            )
 
         if innodb_buffer_pool_size > total_memory:
             log("innodb_buffer_pool_size; {} is greater than system available memory:{}".format(

charmhelpers/contrib/hardening/templating.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import os
+import six
 
 from charmhelpers.core.hookenv import (
     log,
@@ -26,7 +27,10 @@ except ImportError:
     from charmhelpers.fetch import apt_install
     from charmhelpers.fetch import apt_update
     apt_update(fatal=True)
-    apt_install('python-jinja2', fatal=True)
+    if six.PY2:
+        apt_install('python-jinja2', fatal=True)
+    else:
+        apt_install('python3-jinja2', fatal=True)
     from jinja2 import FileSystemLoader, Environment
 
 

charmhelpers/contrib/network/ip.py

@@ -31,14 +31,20 @@ try:
     import netifaces
 except ImportError:
     apt_update(fatal=True)
-    apt_install('python-netifaces', fatal=True)
+    if six.PY2:
+        apt_install('python-netifaces', fatal=True)
+    else:
+        apt_install('python3-netifaces', fatal=True)
     import netifaces
 
 try:
     import netaddr
 except ImportError:
     apt_update(fatal=True)
-    apt_install('python-netaddr', fatal=True)
+    if six.PY2:
+        apt_install('python-netaddr', fatal=True)
+    else:
+        apt_install('python3-netaddr', fatal=True)
     import netaddr
 
 
@@ -414,7 +420,10 @@ def ns_query(address):
     try:
         import dns.resolver
     except ImportError:
-        apt_install('python-dnspython', fatal=True)
+        if six.PY2:
+            apt_install('python-dnspython', fatal=True)
+        else:
+            apt_install('python3-dnspython', fatal=True)
         import dns.resolver
 
     if isinstance(address, dns.name.Name):
@@ -462,7 +471,10 @@ def get_hostname(address, fqdn=True):
         try:
             import dns.reversename
         except ImportError:
-            apt_install("python-dnspython", fatal=True)
+            if six.PY2:
+                apt_install("python-dnspython", fatal=True)
+            else:
+                apt_install("python3-dnspython", fatal=True)
             import dns.reversename
 
         rev = dns.reversename.from_address(address)

charmhelpers/contrib/openstack/ha/utils.py

@@ -126,3 +126,14 @@ def assert_charm_supports_dns_ha():
         status_set('blocked', msg)
         raise DNSHAException(msg)
     return True
+
+
+def expect_ha():
+    """ Determine if the unit expects to be in HA
+
+    Check for VIP or dns-ha settings which indicate the unit should expect to
+    be related to hacluster.
+
+    @returns boolean
+    """
+    return config('vip') or config('dns-ha')

charmhelpers/contrib/openstack/utils.py

@@ -153,7 +153,7 @@ SWIFT_CODENAMES = OrderedDict([
     ('newton',
         ['2.8.0', '2.9.0', '2.10.0']),
     ('ocata',
-        ['2.11.0', '2.12.0']),
+        ['2.11.0', '2.12.0', '2.13.0']),
 ])
 
 # >= Liberty version->codename mapping

charmhelpers/contrib/python/packages.py

@@ -16,6 +16,7 @@
 # limitations under the License.
 
 import os
+import six
 import subprocess
 import sys
 
@@ -39,7 +40,10 @@ def pip_execute(*args, **kwargs):
             from pip import main as _pip_execute
         except ImportError:
             apt_update()
-            apt_install('python-pip')
+            if six.PY2:
+                apt_install('python-pip')
+            else:
+                apt_install('python3-pip')
             from pip import main as _pip_execute
         _pip_execute(*args, **kwargs)
     finally:
@@ -136,7 +140,10 @@ def pip_list():
 
 def pip_create_virtualenv(path=None):
     """Create an isolated Python environment."""
-    apt_install('python-virtualenv')
+    if six.PY2:
+        apt_install('python-virtualenv')
+    else:
+        apt_install('python3-virtualenv')
 
     if path:
         venv_path = path

config.yaml

@@ -37,9 +37,9 @@ options:
     default:
     description: |
       By default this value will be set according to 50% of system total
-      memory but also can be set to any specific value for the system.
-      Supported suffixes include K/M/G/T. If suffixed with %, one will get that
-      percentage of system total memory allocated.
+      memory or 512MB (whichever is lowest) but also can be set to any specific
+      value for the system.  Supported suffixes include K/M/G/T. If suffixed
+      with %, one will get that percentage of system total memory allocated.
   max-connections:
     type: int
     default: 600

tests/basic_deployment.py

@@ -42,8 +42,7 @@ class BasicDeployment(OpenStackAmuletDeployment):
            compatible with the local charm (e.g. stable or next).
            """
         this_service = {'name': 'percona-cluster',
-                        'units': self.units,
-                        'constraints': {'mem': '3072M'}}
+                        'units': self.units}
         other_services = []
         if self.units > 1:
             other_services.append({'name': 'hacluster'})
@@ -61,7 +60,6 @@ class BasicDeployment(OpenStackAmuletDeployment):
         """Configure all of the services."""
         cfg_percona = {'sst-password': 'ubuntu',
                        'root-password': 't00r',
-                       'dataset-size': '512M',
                        'vip': self.vip}
 
         cfg_ha = {'debug': True,

tests/charmhelpers/contrib/openstack/amulet/utils.py

@@ -32,6 +32,7 @@ from keystoneclient.v3 import client as keystone_client_v3
 from novaclient import exceptions
 
 import novaclient.client as nova_client
+import novaclient
 import pika
 import swiftclient
 
@@ -434,9 +435,14 @@ class OpenStackAmuletUtils(AmuletUtils):
         self.log.debug('Authenticating nova user ({})...'.format(user))
         ep = keystone.service_catalog.url_for(service_type='identity',
                                               endpoint_type='publicURL')
-        return nova_client.Client(NOVA_CLIENT_VERSION,
-                                  username=user, api_key=password,
-                                  project_id=tenant, auth_url=ep)
+        if novaclient.__version__[0] >= "7":
+            return nova_client.Client(NOVA_CLIENT_VERSION,
+                                      username=user, password=password,
+                                      project_name=tenant, auth_url=ep)
+        else:
+            return nova_client.Client(NOVA_CLIENT_VERSION,
+                                      username=user, api_key=password,
+                                      project_id=tenant, auth_url=ep)
 
     def authenticate_swift_user(self, keystone, user, password, tenant):
         """Authenticates a regular user with swift api."""