Add python3 support

Not all projects support python3 yet (swift being the worst offender)
and some only have partial support. Nova doesnt support python3 with xen
yet, but it does for everything else.

With this patch a full python3 deploy of openstack *should* be possible.
For now, at least it all builds.

Change-Id: Id03bbae1063b21e949085a3ef8f8c7534d2db3ff

Dockerfile

@@ -10,6 +10,7 @@ ARG DISTRO
 ARG PROFILES
 ARG PIP_PACKAGES
 ARG PLUGIN=no
+ARG PYTHON3=no
 
 COPY scripts /opt/loci/scripts
 COPY bindep.txt /opt/loci/

bindep.txt

@@ -30,7 +30,8 @@ libmariadbclient-dev        [platform:debian requirements]
 libmysqlclient-dev          [platform:ubuntu requirements]
 libnss3-dev                 [platform:dpkg requirements]
 libpq-dev                   [platform:dpkg requirements]
-libpython2.7                [platform:dpkg neutron nova]
+libpython2.7                [platform:dpkg !python3]
+libpython3.5                [platform:dpkg python3]
 libsasl2-dev                [platform:dpkg requirements]
 libssl-dev                  [platform:dpkg requirements]
 libsystemd-dev              [platform:dpkg requirements]
@@ -59,9 +60,12 @@ openvswitch-switch          [platform:dpkg (neutron openvswitch) (nova openvswit
 pkg-config                  [platform:dpkg requirements]
 pkgconfig                   [platform:rpm requirements]
 postgresql-devel            [platform:rpm requirements]
-python-devel                [platform:rpm requirements]
-python-dev                  [platform:dpkg requirements]
-python-rbd                  [(ceph glance)]
+python-devel                [platform:rpm (requirements !python3)]
+python3-devel               [platform:rpm (requirements python3)]
+python-dev                  [platform:dpkg (requirements !python3)]
+python3-dev                 [platform:dpkg (requirements python3)]
+python-rbd                  [(ceph glance) !python3]
+python3-rbd                 [(ceph glance) python3]
 qemu-img                    [platform:rpm (qemu nova)]
 qemu-utils                  [platform:dpkg (qemu nova)]
 systemd-devel               [platform:rpm requirements]

scripts/cleanup.sh

@@ -6,13 +6,15 @@ case ${distro} in
     debian|ubuntu)
         apt-get purge -y --auto-remove \
             git \
+            python3-virtualenv \
             virtualenv
         rm -rf /var/lib/apt/lists/*
         ;;
     centos)
         yum -y autoremove \
             git \
-            python-virtualenv
+            python-virtualenv \
+            python3-virtualenv
         yum clean all
         ;;
     *)
@@ -27,4 +29,4 @@ esac
 # packaged form.
 rm /var/lib/openstack/lib/python*/no-global-site-packages.txt
 rm -rf /tmp/* /root/.cache
-find /usr/ /var/ -type f -name "*.pyc" -delete
+find /usr/ /var/ \( -name "*.pyc" -o -name "__pycache__" \) -delete

scripts/fetch_wheels.sh

@@ -2,7 +2,13 @@
 
 set -ex
 
-$(dirname $0)/fetch_wheels.py
+if [[ "${PYTHON3}" == "no" ]]; then
+    python=python2
+else
+    python=python3
+fi
+
+${python} $(dirname $0)/fetch_wheels.py
 
 mkdir -p /tmp/wheels/
 # NOTE(SamYaple): We exclude all files starting with '.' as these can be

scripts/install.sh

@@ -5,6 +5,14 @@ set -ex
 distro=$(awk -F= '/^ID=/ {gsub(/\"/, "", $2); print $2}' /etc/*release)
 export distro=${DISTRO:=$distro}
 
+if [[ "${PYTHON3}" == "no" ]]; then
+    dpkg_python_packages=("python" "virtualenv")
+    rpm_python_packages=("python" "python-virtualenv")
+else
+    dpkg_python_packages=("python3" "python3-virtualenv")
+    rpm_python_packages=("python3" "python3-virtualenv")
+fi
+
 case ${distro} in
     debian|ubuntu)
         apt-get update
@@ -13,19 +21,17 @@ case ${distro} in
             git \
             ca-certificates \
             netbase \
-            python \
-            virtualenv \
             lsb-release \
-            sudo
+            sudo \
+            ${dpkg_python_packages[@]}
         ;;
     centos)
         yum upgrade -y
         yum install -y --setopt=skip_missing_names_on_install=False \
             git \
-            python \
-            python-virtualenv \
             redhat-lsb-core \
-            sudo
+            sudo \
+            ${rpm_python_packages[@]}
         ;;
     *)
         echo "Unknown distro: ${distro}"

scripts/install_packages.sh

@@ -2,7 +2,11 @@
 
 set -ex
 
-PACKAGES=($(bindep -f /opt/loci/bindep.txt -b ${PROJECT} ${PROFILES} || :))
+if [[ "${PYTHON3}" != "no" ]]; then
+    python3=python3
+fi
+
+PACKAGES=($(bindep -f /opt/loci/bindep.txt -b ${PROJECT} ${PROFILES} ${python3} || :))
 
 if [[ ! -z ${PACKAGES} ]]; then
     case ${distro} in

scripts/setup_pip.sh

@@ -2,13 +2,19 @@
 
 set -ex
 
+if [[ "${PYTHON3}" == "no" ]]; then
+    TMP_VIRTUALENV="virtualenv"
+else
+    TMP_VIRTUALENV="python3 -m virtualenv --python=python3"
+fi
+
 # NOTE(SamYaple): This little dance allows us to install the latest pip and
 # setuptools without get_pip.py or the python-pip package (which is in epel on
 # centos)
-if (( $(virtualenv --version | cut -d. -f1) >= 14 )); then
+if (( $(${TMP_VIRTUALENV} --version | cut -d. -f1) >= 14 )); then
     SETUPTOOLS="--no-setuptools"
 fi
-virtualenv --extra-search-dir=/tmp/wheels ${SETUPTOOLS} /tmp/venv
+${TMP_VIRTUALENV} --extra-search-dir=/tmp/wheels ${SETUPTOOLS} /tmp/venv
 source /tmp/venv/bin/activate
 pip install --upgrade virtualenv
 hash -r