Charm-helper sync to pull in dns resolve fix

Change d5d2e5064e introduced this fix
However the charm-helpers branch was not merged. This is a
reintroduction of the same code with a charm-helpers sync after the
charm-helpers branch was merged.

Change-Id: I68f53d5d4ed726c14d8cc83c650763fa77a4324d

hooks/charmhelpers/contrib/network/ip.py

@@ -424,7 +424,11 @@ def ns_query(address):
     else:
         return None
 
-    answers = dns.resolver.query(address, rtype)
+    try:
+        answers = dns.resolver.query(address, rtype)
+    except dns.resolver.NXDOMAIN as e:
+        return None
+
     if answers:
         return str(answers[0])
     return None

hooks/charmhelpers/contrib/openstack/context.py

@@ -14,6 +14,7 @@
 
 import glob
 import json
+import math
 import os
 import re
 import time
@@ -90,6 +91,8 @@ from charmhelpers.contrib.network.ip import (
 from charmhelpers.contrib.openstack.utils import (
     config_flags_parser,
     get_host_ip,
+    git_determine_usr_bin,
+    git_determine_python_path,
     enable_memcache,
 )
 from charmhelpers.core.unitdata import kv
@@ -1208,6 +1211,43 @@ class WorkerConfigContext(OSContextGenerator):
         return ctxt
 
 
+class WSGIWorkerConfigContext(WorkerConfigContext):
+
+    def __init__(self, name=None, script=None, admin_script=None,
+                 public_script=None, process_weight=1.00,
+                 admin_process_weight=0.75, public_process_weight=0.25):
+        self.service_name = name
+        self.user = name
+        self.group = name
+        self.script = script
+        self.admin_script = admin_script
+        self.public_script = public_script
+        self.process_weight = process_weight
+        self.admin_process_weight = admin_process_weight
+        self.public_process_weight = public_process_weight
+
+    def __call__(self):
+        multiplier = config('worker-multiplier') or 1
+        total_processes = self.num_cpus * multiplier
+        ctxt = {
+            "service_name": self.service_name,
+            "user": self.user,
+            "group": self.group,
+            "script": self.script,
+            "admin_script": self.admin_script,
+            "public_script": self.public_script,
+            "processes": int(math.ceil(self.process_weight * total_processes)),
+            "admin_processes": int(math.ceil(self.admin_process_weight *
+                                             total_processes)),
+            "public_processes": int(math.ceil(self.public_process_weight *
+                                              total_processes)),
+            "threads": 1,
+            "usr_bin": git_determine_usr_bin(),
+            "python_path": git_determine_python_path(),
+        }
+        return ctxt
+
+
 class ZeroMQContext(OSContextGenerator):
     interfaces = ['zeromq-configuration']
 
@@ -1521,9 +1561,18 @@ class MemcacheContext(OSContextGenerator):
     This context provides options for configuring a local memcache client and
     server
     """
+
+    def __init__(self, package=None):
+        """
+        @param package: Package to examine to extrapolate OpenStack release.
+                        Used when charms have no openstack-origin config
+                        option (ie subordinates)
+        """
+        self.package = package
+
     def __call__(self):
         ctxt = {}
-        ctxt['use_memcache'] = enable_memcache(config('openstack-origin'))
+        ctxt['use_memcache'] = enable_memcache(package=self.package)
         if ctxt['use_memcache']:
             # Trusty version of memcached does not support ::1 as a listen
             # address so use host file entry instead

hooks/charmhelpers/contrib/openstack/templates/wsgi-openstack-api.conf

@@ -0,0 +1,100 @@
+# Configuration file maintained by Juju. Local changes may be overwritten.
+
+{% if port -%}
+Listen {{ port }}
+{% endif -%}
+
+{% if admin_port -%}
+Listen {{ admin_port }}
+{% endif -%}
+
+{% if public_port -%}
+Listen {{ public_port }}
+{% endif -%}
+
+{% if port -%}
+<VirtualHost *:{{ port }}>
+    WSGIDaemonProcess {{ service_name }} processes={{ processes }} threads={{ threads }} user={{ service_name }} group={{ service_name }} \
+{% if python_path -%}
+                      python-path={{ python_path }} \
+{% endif -%}
+                      display-name=%{GROUP}
+    WSGIProcessGroup {{ service_name }}
+    WSGIScriptAlias / {{ script }}
+    WSGIApplicationGroup %{GLOBAL}
+    WSGIPassAuthorization On
+    <IfVersion >= 2.4>
+      ErrorLogFormat "%{cu}t %M"
+    </IfVersion>
+    ErrorLog /var/log/apache2/{{ service_name }}_error.log
+    CustomLog /var/log/apache2/{{ service_name }}_access.log combined
+
+    <Directory {{ usr_bin }}>
+        <IfVersion >= 2.4>
+            Require all granted
+        </IfVersion>
+        <IfVersion < 2.4>
+            Order allow,deny
+            Allow from all
+        </IfVersion>
+    </Directory>
+</VirtualHost>
+{% endif -%}
+
+{% if admin_port -%}
+<VirtualHost *:{{ admin_port }}>
+    WSGIDaemonProcess {{ service_name }}-admin processes={{ admin_processes }} threads={{ threads }} user={{ service_name }} group={{ service_name }} \
+{% if python_path -%}
+                      python-path={{ python_path }} \
+{% endif -%}
+                      display-name=%{GROUP}
+    WSGIProcessGroup {{ service_name }}-admin
+    WSGIScriptAlias / {{ admin_script }}
+    WSGIApplicationGroup %{GLOBAL}
+    WSGIPassAuthorization On
+    <IfVersion >= 2.4>
+      ErrorLogFormat "%{cu}t %M"
+    </IfVersion>
+    ErrorLog /var/log/apache2/{{ service_name }}_error.log
+    CustomLog /var/log/apache2/{{ service_name }}_access.log combined
+
+    <Directory {{ usr_bin }}>
+        <IfVersion >= 2.4>
+            Require all granted
+        </IfVersion>
+        <IfVersion < 2.4>
+            Order allow,deny
+            Allow from all
+        </IfVersion>
+    </Directory>
+</VirtualHost>
+{% endif -%}
+
+{% if public_port -%}
+<VirtualHost *:{{ public_port }}>
+    WSGIDaemonProcess {{ service_name }}-public processes={{ public_processes }} threads={{ threads }} user={{ service_name }} group={{ service_name }} \
+{% if python_path -%}
+                      python-path={{ python_path }} \
+{% endif -%}
+                      display-name=%{GROUP}
+    WSGIProcessGroup {{ service_name }}-public
+    WSGIScriptAlias / {{ public_script }}
+    WSGIApplicationGroup %{GLOBAL}
+    WSGIPassAuthorization On
+    <IfVersion >= 2.4>
+      ErrorLogFormat "%{cu}t %M"
+    </IfVersion>
+    ErrorLog /var/log/apache2/{{ service_name }}_error.log
+    CustomLog /var/log/apache2/{{ service_name }}_access.log combined
+
+    <Directory {{ usr_bin }}>
+        <IfVersion >= 2.4>
+            Require all granted
+        </IfVersion>
+        <IfVersion < 2.4>
+            Order allow,deny
+            Allow from all
+        </IfVersion>
+    </Directory>
+</VirtualHost>
+{% endif -%}

hooks/charmhelpers/contrib/openstack/utils.py

@@ -1119,6 +1119,35 @@ def git_generate_systemd_init_files(templates_dir):
                 shutil.copyfile(service_source, service_dest)
 
 
+def git_determine_usr_bin():
+    """Return the /usr/bin path for Apache2 config.
+
+    The /usr/bin path will be located in the virtualenv if the charm
+    is configured to deploy from source.
+    """
+    if git_install_requested():
+        projects_yaml = config('openstack-origin-git')
+        projects_yaml = git_default_repos(projects_yaml)
+        return os.path.join(git_pip_venv_dir(projects_yaml), 'bin')
+    else:
+        return '/usr/bin'
+
+
+def git_determine_python_path():
+    """Return the python-path for Apache2 config.
+
+    Returns 'None' unless the charm is configured to deploy from source,
+    in which case the path of the virtualenv's site-packages is returned.
+    """
+    if git_install_requested():
+        projects_yaml = config('openstack-origin-git')
+        projects_yaml = git_default_repos(projects_yaml)
+        return os.path.join(git_pip_venv_dir(projects_yaml),
+                            'lib/python2.7/site-packages')
+    else:
+        return None
+
+
 def os_workload_status(configs, required_interfaces, charm_func=None):
     """
     Decorator to set workload status based on complete contexts
@@ -1927,16 +1956,24 @@ def os_application_version_set(package):
         application_version_set(application_version)
 
 
-def enable_memcache(source=None, release=None):
+def enable_memcache(source=None, release=None, package=None):
     """Determine if memcache should be enabled on the local unit
 
-    @param source: source string for charm
     @param release: release of OpenStack currently deployed
+    @param package: package to derive OpenStack version deployed
     @returns boolean Whether memcache should be enabled
     """
-    if not release:
-        release = get_os_codename_install_source(source)
-    return release >= 'mitaka'
+    _release = None
+    if release:
+        _release = release
+    else:
+        _release = os_release(package, base='icehouse')
+    if not _release:
+        _release = get_os_codename_install_source(source)
+
+    # TODO: this should be changed to a numeric comparison using a known list
+    # of releases and comparing by index.
+    return _release >= 'mitaka'
 
 
 def token_cache_pkgs(source=None, release=None):

hooks/charmhelpers/core/hookenv.py

@@ -616,6 +616,20 @@ def close_port(port, protocol="TCP"):
     subprocess.check_call(_args)
 
 
+def open_ports(start, end, protocol="TCP"):
+    """Opens a range of service network ports"""
+    _args = ['open-port']
+    _args.append('{}-{}/{}'.format(start, end, protocol))
+    subprocess.check_call(_args)
+
+
+def close_ports(start, end, protocol="TCP"):
+    """Close a range of service network ports"""
+    _args = ['close-port']
+    _args.append('{}-{}/{}'.format(start, end, protocol))
+    subprocess.check_call(_args)
+
+
 @cached
 def unit_get(attribute):
     """Get the unit ID for the remote unit"""

hooks/charmhelpers/core/host.py

@@ -306,15 +306,17 @@ def add_user_to_group(username, group):
     subprocess.check_call(cmd)
 
 
-def rsync(from_path, to_path, flags='-r', options=None):
+def rsync(from_path, to_path, flags='-r', options=None, timeout=None):
     """Replicate the contents of a path"""
     options = options or ['--delete', '--executability']
     cmd = ['/usr/bin/rsync', flags]
+    if timeout:
+        cmd = ['timeout', str(timeout)] + cmd
     cmd.extend(options)
     cmd.append(from_path)
     cmd.append(to_path)
     log(" ".join(cmd))
-    return subprocess.check_output(cmd).decode('UTF-8').strip()
+    return subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode('UTF-8').strip()
 
 
 def symlink(source, destination):
@@ -684,7 +686,7 @@ def chownr(path, owner, group, follow_links=True, chowntopdir=False):
     :param str path: The string path to start changing ownership.
     :param str owner: The owner string to use when looking up the uid.
     :param str group: The group string to use when looking up the gid.
-    :param bool follow_links: Also Chown links if True
+    :param bool follow_links: Also follow and chown links if True
     :param bool chowntopdir: Also chown path itself if True
     """
     uid = pwd.getpwnam(owner).pw_uid
@@ -698,7 +700,7 @@ def chownr(path, owner, group, follow_links=True, chowntopdir=False):
         broken_symlink = os.path.lexists(path) and not os.path.exists(path)
         if not broken_symlink:
             chown(path, uid, gid)
-    for root, dirs, files in os.walk(path):
+    for root, dirs, files in os.walk(path, followlinks=follow_links):
         for name in dirs + files:
             full = os.path.join(root, name)
             broken_symlink = os.path.lexists(full) and not os.path.exists(full)