Pegleg focal upgrade

This PS delivers focal version of Pegleg image and has the following updates:
- removed release-notes-jobs-python3 gate job because of incompartibility with Sphinx from current requirements
- added focal gate node and switched gates to use it
- added bindep.txt file into project root
- added bindep role into gate jobs
- added ubuntu_focal dockerfile for building focal pegleg image
- switched tox profiles to py38
- uplifted references to shipyard_client, promenade and deckhand projects
- resolved required dependencies conflicts by weakening constraints in Pipfile
- updated tox profile update-requirements for generate requirements.txt and test-requirements.txt
- generated new Pipfile.lock, requirements.txt and test-requirements.txt from Pipfile
- switched tox profiles to use requirements.txt and test-requirements.txt instead of pipenv because of upstream zuul nodes Pypi mirrorring issue
- updated reference to seaworthy site certificates in treasuremap repo
- fixed unit tests issues caused by pytest/mock updates and new openssl version
- fixed focal docker image publishing issue
- added multiprocessing into coverage tests running process
- made unit and coveraget tests more verbosive

Change-Id: I5c4c519dc725cfb8c7b4e14756347c9336028aff
This commit is contained in:
Sergiy Markin 2023-04-28 16:13:30 +00:00
parent 770cf4256e
commit c052d40277
21 changed files with 2248 additions and 1054 deletions

View File

@ -13,9 +13,9 @@
- project:
templates:
- docs-on-readthedocs
- openstack-python36-jobs
- openstack-python38-jobs
- openstack-cover-jobs
- release-notes-jobs-python3
# - release-notes-jobs-python3
vars:
rtd_webhook_id: '38574'
rtd_project_name: 'airship-pegleg'
@ -26,25 +26,26 @@
docker_registry_login_url: "https://quay.io/api/v1/"
deb_docker_repo: "{{ zuul_site_mirror_fqdn }}/deb-docker/{{ ansible_distribution_release }}"
image_prefix: "airshipit"
base_image_focal: "ubuntu:20.04"
base_image_bionic: "ubuntu:18.04"
base_image_opensuse: "opensuse/leap:15.1"
base_image_xenial: "ubuntu:16.04"
check:
jobs:
- openstack-tox-pep8-bionic
- openstack-tox-pep8-focal
- pegleg-dependency-vulnerability-check
- airship-pegleg-docker-build-gate-ubuntu_bionic
- airship-pegleg-docker-build-gate-ubuntu_focal
- airship-pegleg-lint-yaml
gate:
jobs:
- openstack-tox-pep8-bionic
- openstack-tox-pep8-focal
- pegleg-dependency-vulnerability-check
- airship-pegleg-docker-build-gate-ubuntu_bionic
- airship-pegleg-docker-build-gate-ubuntu_focal
- airship-pegleg-lint-yaml
post:
jobs:
- airship-pegleg-docker-publish-ubuntu_bionic
- airship-pegleg-docker-publish-ubuntu_focal
- pegleg-upload-git-mirror
- nodeset:
@ -54,62 +55,62 @@
label: ubuntu-xenial
- nodeset:
name: airship-pegleg-single-node-bionic
name: airship-pegleg-single-node-focal
nodes:
- name: primary
label: ubuntu-bionic
label: ubuntu-focal
- job:
name: openstack-tox-pep8-bionic
name: openstack-tox-pep8-focal
parent: openstack-tox-pep8
nodeset: openstack-single-node-bionic
nodeset: openstack-single-node-focal
- job:
name: airship-pegleg-lint-yaml
voting: true
timeout: 600
run: tools/gate/playbooks/lint-yaml.yaml
nodeset: ubuntu-bionic
nodeset: ubuntu-focal
irrelevant-files:
- ^.*\.rst$
- ^doc/.*$
- job:
name: pegleg-dependency-vulnerability-check
parent: tox-py36
parent: tox-py38
voting: false
timeout: 600
nodeset: ubuntu-bionic
nodeset: ubuntu-focal
vars:
tox_envlist: safety
bindep_profile: test py36
bindep_profile: test py38
- job:
name: airship-pegleg-docker-build-gate-ubuntu_bionic
name: airship-pegleg-docker-build-gate-ubuntu_focal
timeout: 1800
run: tools/gate/playbooks/docker-image-build.yaml
nodeset: airship-pegleg-single-node-bionic
nodeset: airship-pegleg-single-node-focal
irrelevant-files:
- '^doc/.*'
vars:
publish: false
distro: ubuntu_bionic
distro: ubuntu_focal
tags:
dynamic:
patch_set: true
- job:
name: airship-pegleg-docker-publish-ubuntu_bionic
name: airship-pegleg-docker-publish-ubuntu_focal
timeout: 1800
run: tools/gate/playbooks/docker-image-build.yaml
nodeset: airship-pegleg-single-node-bionic
nodeset: airship-pegleg-single-node-focal
secrets:
- airship_pegleg_quay_creds
irrelevant-files:
- '^doc/.*'
vars:
publish: true
distro: ubuntu_bionic
distro: ubuntu_focal
tags:
dynamic:
branch: true

View File

@ -27,9 +27,9 @@ PUSH_IMAGE ?= false
# use this variable for image labels added in internal build process
LABEL ?= org.airshipit.build=community
COMMIT ?= $(shell git rev-parse HEAD)
DISTRO ?= ubuntu_bionic
DISTRO ?= ubuntu_focal
IMAGE ?= $(DOCKER_REGISTRY)/$(IMAGE_PREFIX)/$(IMAGE_NAME):$(IMAGE_TAG)-${DISTRO}
PYTHON_BASE_IMAGE ?= python:3.6
PYTHON_BASE_IMAGE ?= python:3.8
BASE_IMAGE ?=
export

40
Pipfile
View File

@ -4,8 +4,9 @@ url = "https://pypi.org/simple"
verify_ssl = true
[dev-packages]
pytest = "~=6.1.1"
pytest-cov = "~=2.10.1"
bandit = "~=1.6.0"
pytest = "*"
pytest-cov = "*"
testfixtures = "*"
pytest-xdist = "*"
requests = ">=2.20.0"
@ -17,22 +18,23 @@ hacking = ">=1.1.0"
flake8-import-order = ">=0.18.1"
[packages]
click = ">=6.7"
jsonschema = ">=3.0.1,<4"
cryptography = ">=2.7"
python-dateutil = ">=2.8.1"
GitPython = "~=2.1.11"
docker = ">=3.7.2"
pylibyaml = "~=0.1"
PyYAML = "~=5.1"
deckhand = {git = "https://opendev.org/airship/deckhand.git",ref = "5cd799cc5d04527ac782270008ff647b3779ff05"}
shipyard-client = {git = "https://opendev.org/airship/shipyard.git",ref = "e3e71f7d1632c0dc13b436a32c238f9da76d6d88",subdirectory = "src/bin/shipyard_client"}
promenade = {git = "https://opendev.org/airship/promenade.git",ref = "c10165c144e0a18137596a3c89f1339d6ed30d0c"}
"oslo.i18n" = "~=3.24.0"
"oslo.utils" = ">=3.42.1"
six = ">=1.15.0"
debtcollector = "~=1.22.0"
zipp = "~=3.6.0"
click = "*"
jsonschema = "*"
cryptography = "*"
python-dateutil = "*"
GitPython = "*"
docker = "*"
pylibyaml = "*"
PyYAML = "*"
deckhand = {git = "https://opendev.org/airship/deckhand.git",ref = "03f6932e16aa0b72a32a10fc04a52125c45dd5d7"}
shipyard-client = {git = "https://opendev.org/airship/shipyard.git",ref = "154a099b2843b1b42a7b8afa5fa98383cf3c1b7c",subdirectory = "src/bin/shipyard_client"}
promenade = {git = "https://opendev.org/airship/promenade.git",ref = "97e3721a03ca48d312891694972b827a53714bf9"}
"oslo.i18n" = "*"
"oslo.utils" = "*"
six = "*"
debtcollector = "*"
zipp = "*"
psycopg2-binary = "*"
[requires]
python_version = "3.6"
python_version = "3.8"

2445
Pipfile.lock generated

File diff suppressed because it is too large Load Diff

10
bindep.txt Normal file
View File

@ -0,0 +1,10 @@
# This is a cross-platform list tracking distribution packages needed by tests;
# see http://docs.openstack.org/infra/bindep/ for additional information.
libffi-dev [test platform:dpkg]
libkrb5-dev [platform:dpkg]
libpq-dev [platform:dpkg]
libsasl2-dev [platform:dpkg]
libssl-dev [platform:dpkg]
libre2-dev [platform:dpkg]
libzmq3-dev [platform:dpkg]

View File

@ -3,13 +3,15 @@
# process, which may cause wedges in the gate later.
# Documentation
sphinx>=3.1.0
oslosphinx>=4.18.0 # Apache-2.0
sphinx_rtd_theme>=0.4.3
docutils>=0.17.1
sphinx>=3.3.1
oslosphinx # Apache-2.0
sphinx_rtd_theme>=0.5.0
docutils>=0.17
# UML image generation
plantuml>=0.3.0
plantuml>=
# Releasenotes
reno>=3.1.0 # Apache-2.0
reno>=4.0.0 # Apache-2.0

Binary file not shown.

Before

Width:  |  Height:  |  Size: 37 KiB

After

Width:  |  Height:  |  Size: 42 KiB

View File

@ -0,0 +1,75 @@
ARG FROM=ubuntu:20.04
FROM ${FROM}
ARG CFSSLURL=https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode'
LABEL org.opencontainers.image.url='https://airshipit.org'
LABEL org.opencontainers.image.documentation='https://airship-pegleg.readthedocs.org'
LABEL org.opencontainers.image.source='https://opendev.org/airship/pegleg'
LABEL org.opencontainers.image.vendor='The Airship Authors'
LABEL org.opencontainers.image.licenses='Apache-2.0'
ENV LANG=C.UTF-8
ENV LC_ALL=C.UTF-8
ARG DECKHAND_VERSION=branch/master
ARG SHIPYARD_VERSION=branch/master
ARG PROMENADE_VERSION=branch/master
RUN set -ex \
&& apt update -qq \
&& apt install -y --no-install-recommends \
automake \
ca-certificates \
curl \
build-essential \
git \
gpg \
gpg-agent \
libssl-dev \
libtool \
make \
netbase \
openssh-client \
python3-dev \
python3-pip \
python3-setuptools \
&& python3 -m pip install -U pip \
&& apt autoremove -yqq --purge \
&& apt clean \
&& rm -rf \
/tmp/* \
/usr/share/doc \
/usr/share/doc-base \
/usr/share/man \
/var/lib/apt/lists/* \
/var/log/* \
/var/tmp/*
ENV LD_LIBRARY_PATH=/usr/local/lib
ARG LIBYAML_VERSION=0.2.5
RUN set -ex \
&& git clone https://github.com/yaml/libyaml.git \
&& cd libyaml \
&& git checkout $LIBYAML_VERSION \
&& ./bootstrap \
&& ./configure \
&& make \
&& make install \
&& cd .. \
&& rm -fr libyaml
VOLUME /var/pegleg
WORKDIR /var/pegleg
COPY requirements.txt /opt/pegleg/requirements.txt
RUN pip3 install -r https://opendev.org/airship/deckhand/raw/${DECKHAND_VERSION}/requirements.txt \
&& pip3 install -r https://opendev.org/airship/promenade/raw/${PROMENADE_VERSION}/requirements.txt \
&& pip3 install -r https://opendev.org/airship/shipyard/raw/${SHIPYARD_VERSION}/src/bin/shipyard_client/requirements.txt \
&& pip3 install --no-cache-dir -r /opt/pegleg/requirements.txt
COPY tools/install-cfssl.sh /opt/pegleg/tools/install-cfssl.sh
RUN /opt/pegleg/tools/install-cfssl.sh ${CFSSLURL}
COPY . /opt/pegleg
RUN pip3 install -e /opt/pegleg --use-pep517

View File

@ -1,126 +1,175 @@
-i https://pypi.org/simple
alembic==1.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
amqp==2.6.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
argparse==1.4.0
asn1crypto==1.3.0
babel==2.8.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
beaker==1.10.0
cachetools==4.1.0; python_version ~= '3.5'
certifi==2020.4.5.2
cffi==1.14.0
alabaster==0.7.13 ; python_version >= '3.6'
alembic==1.4.3 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
amqp==2.6.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
argcomplete==3.0.8 ; python_version >= '3.6'
arrow==0.17.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
attrs==23.1.0 ; python_version >= '3.7'
autopage==0.5.1 ; python_version >= '3.6'
babel==2.12.1 ; python_version >= '3.7'
bcrypt==4.0.1 ; python_version >= '3.6'
beaker==1.12.0
cachetools==5.3.0 ; python_version ~= '3.7'
certifi==2022.12.7 ; python_version >= '3.6'
cffi==1.15.1
chardet==3.0.4
charset-normalizer==3.1.0 ; python_full_version >= '3.7.0'
click==6.7
cliff==3.1.0; python_version >= '3.5'
cmd2==0.8.9
cryptography>=2.7
debtcollector==1.22.0
decorator==4.4.2
deepdiff==3.3.0
dnspython==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
docker-pycreds==0.4.0
docker==3.7.2
dogpile.cache==0.9.2
eventlet==0.25.2
click-default-group==1.2
cliff==3.10.1 ; python_version >= '3.6'
cmd2==2.4.3 ; python_version >= '3.6'
configparser==5.3.0 ; python_version >= '3.7'
coverage==7.2.5 ; python_version >= '3.7'
cryptography==3.4.8
debtcollector==2.5.0
git+https://opendev.org/airship/deckhand.git@13c5199f18664d66ccdd3a1e54bb40ad2c293d1a#egg=deckhand
decorator==5.1.1 ; python_version >= '3.5'
deepdiff==5.8.1 ; python_version >= '3.6'
distlib==0.3.6
dnspython==2.3.0 ; python_version >= '3.7' and python_version < '4.0'
docker==5.0.3
docutils==0.17 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
dogpile.cache==1.2.0 ; python_version >= '3.6'
dulwich==0.21.3 ; python_version >= '3.7'
eventlet==0.33.3
extras==1.0.0
falcon==1.4.1
fasteners==0.15
falcon==3.1.1 ; python_version >= '3.5'
fasteners==0.18 ; python_version >= '3.6'
filelock==3.12.0 ; python_version >= '3.7'
fixtures==3.0.0
flake8==3.7.9
futurist==2.2.0; python_version >= '3.6'
git+https://opendev.org/airship/deckhand.git@5cd799cc5d04527ac782270008ff647b3779ff05#egg=deckhand
git+https://opendev.org/airship/promenade.git@c10165c144e0a18137596a3c89f1339d6ed30d0c#egg=promenade
git+https://opendev.org/airship/shipyard.git@e3e71f7d1632c0dc13b436a32c238f9da76d6d88#egg=shipyard-client&subdirectory=src/bin/shipyard_client
gitpython==2.1.15
setuptools==40.4.3
google-auth==1.16.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
greenlet==0.4.16
hacking>=3.0.1,<3.1.0
idna==2.8
importlib-metadata==1.6.1; python_version < '3.8'
ipaddress==1.0.23
iso8601==0.1.12
jinja2==2.10
jsonpath-ng==1.4.3
jsonpickle==1.4.1; python_version >= '2.7'
jsonschema>=3.0.1,<4
keystoneauth1>=3.18.0
keystonemiddleware==5.3.0
kombu==4.6.10; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
kubernetes~=24.2.0; python_version >= '3.6'
linecache2==1.0.0
mako==1.1.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
markupsafe==1.1.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
flake8==3.8.4 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
futurist==2.4.1 ; python_version >= '3.6'
gitdb==4.0.10 ; python_version >= '3.7'
gitpython==3.1.31
google-auth==2.17.3 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
greenlet==2.0.2 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
hacking==4.1.0 ; python_version >= '3.5'
html5lib==0.9999999
httpexceptor==1.4.0
idna==2.10 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
imagesize==1.4.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
importlib-metadata==2.1.3 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
iso8601==1.1.0 ; python_full_version >= '3.6.2' and python_version < '4.0'
jinja2==3.0.3 ; python_version >= '3.6'
jsonpath-ng==1.5.3
jsonpath-rw==1.4.0
jsonpath-rw-ext==1.2.2
jsonpickle==1.4.1 ; python_version >= '2.7'
jsonschema==3.2.0
keystoneauth1==5.1.1
keystonemiddleware==10.2.0 ; python_version >= '3.8'
kombu==4.6.11 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
kubernetes==26.1.0 ; python_version >= '3.6'
mako==1.2.4 ; python_version >= '3.7'
markupsafe==2.0.1 ; python_version >= '3.6'
mccabe==0.6.1
monotonic==1.5
msgpack==1.0.0
netaddr==0.7.19
netifaces==0.10.9
networkx==2.2
mock==5.0.2 ; python_version >= '3.6'
msgpack==1.0.5
netaddr==0.8.0
netifaces==0.11.0
networkx==3.1 ; python_version >= '3.8'
nose==1.3.7
oauthlib==3.2.2 ; python_version >= '3.6'
ordered-set==4.1.0 ; python_version >= '3.7'
os-service-types==1.7.0
oslo.cache==1.38.1
oslo.concurrency==3.28.1
oslo.config==7.0.0
oslo.context==2.21.0
oslo.db==4.41.1
oslo.i18n==3.24.0
oslo.log==3.45.2
oslo.messaging==9.1.1
oslo.middleware==3.36.0
oslo.policy==1.40.1
oslo.serialization==2.29.2
oslo.service==2.2.0; python_version >= '3.6'
oslo.utils==3.42.1
paste==3.0.1
pastedeploy==1.5.2
pbr==5.4.5
oslo.cache==2.10.1 ; python_version >= '3.6'
oslo.concurrency==5.1.1 ; python_version >= '3.8'
oslo.config==8.7.1 ; python_version >= '3.6'
oslo.context==4.1.0 ; python_version >= '3.6'
oslo.db==10.0.0 ; python_version >= '3.6'
oslo.i18n==6.0.0
oslo.log==4.6.0 ; python_version >= '3.6'
oslo.messaging==12.13.0 ; python_version >= '3.6'
oslo.metrics==0.6.0 ; python_version >= '3.8'
oslo.middleware==4.4.0 ; python_version >= '3.6'
oslo.policy==3.10.1 ; python_version >= '3.6'
oslo.serialization==4.2.0 ; python_version >= '3.6'
oslo.service==3.1.1 ; python_version >= '3.8'
oslo.utils==4.12.3
packaging==21.3 ; python_version >= '3.6'
paste==3.5.0
pastedeploy==3.0.1 ; python_version >= '3.7'
pastescript==3.3.0
pbr==5.5.1 ; python_version >= '2.6'
platformdirs==3.5.0 ; python_version >= '3.7'
ply==3.11
prettytable==0.7.2
psycopg2-binary==2.8.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
pyasn1-modules==0.2.8
pyasn1==0.4.8
pycadf==3.0.0; python_version >= '3.6'
pycodestyle==2.5.0
pycparser==2.20; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
pyflakes==2.1.0
pyinotify==0.9.6; sys_platform != 'win32' and sys_platform != 'darwin' and sys_platform != 'sunos5'
prettytable==3.7.0 ; python_version >= '3.7'
git+https://opendev.org/airship/promenade.git@97e3721a03ca48d312891694972b827a53714bf9#egg=promenade
prometheus-client==0.16.0 ; python_version >= '3.6'
psycopg2-binary==2.9.6
ptable==0.9.2
pyasn1==0.5.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
pyasn1-modules==0.3.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
pycadf==3.1.1 ; python_version >= '3.6'
pycodestyle==2.6.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
pycparser==2.21
pyflakes==2.2.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
pygments==2.14.0 ; python_version >= '3.6'
pyinotify==0.9.6 ; sys_platform != 'win32' and sys_platform != 'darwin' and sys_platform != 'sunos5'
pylibyaml==0.1.0
pyparsing==2.4.7; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
pyperclip==1.8.0
python-barbicanclient==4.7.0
python-dateutil>=2.8.1
pymongo==4.3.3 ; python_version >= '3.7'
pyparsing==2.4.7 ; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'
pyperclip==1.8.2
pyproject-api==1.5.0 ; python_version >= '3.7'
pyrsistent==0.19.3 ; python_version >= '3.7'
python-barbicanclient==5.2.0
python-dateutil==2.8.1
python-editor==1.0.4
python-keystoneclient==3.22.0
python-keystoneclient==5.1.0 ; python_version >= '3.8'
python-memcached==1.59
python-mimeparse==1.6.0
pytz==2020.1
pyyaml~=5.1
python-subunit==1.4.0
pytz==2023.3
pyyaml==5.4.1
reno==4.0.0 ; python_version >= '3.6'
repoze.lru==0.7
requests==2.22.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
rfc3986==1.4.0
routes==2.4.1
rsa==4.0
six>=1.15.0
smmap2==3.0.1
smmap==3.0.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
requests==2.23.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
requests-oauthlib==1.3.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
resolver==0.2.1
rfc3986==2.0.0 ; python_version >= '3.7'
routes==2.5.1
rsa==4.9 ; python_version >= '3.6'
selector==0.10.1
setuptools==45.2.0 ; python_version >= '3.5'
git+https://opendev.org/airship/shipyard.git@154a099b2843b1b42a7b8afa5fa98383cf3c1b7c#egg=shipyard-client&subdirectory=src/bin/shipyard_client
simplejson==3.19.1 ; python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2'
six==1.16.0
smmap==5.0.0 ; python_version >= '3.6'
snowballstemmer==2.2.0
sphinx==3.3.1 ; python_version >= '3.5'
sphinx-rtd-theme==0.5.0
sphinxcontrib-applehelp==1.0.4 ; python_version >= '3.8'
sphinxcontrib-devhelp==1.0.2 ; python_version >= '3.5'
sphinxcontrib-htmlhelp==2.0.1 ; python_version >= '3.8'
sphinxcontrib-jsmath==1.0.1 ; python_version >= '3.5'
sphinxcontrib-qthelp==1.0.3 ; python_version >= '3.5'
sphinxcontrib-serializinghtml==1.1.5 ; python_version >= '3.5'
sqlalchemy==1.3.20 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
sqlalchemy-migrate==0.13.0
sqlalchemy==1.3.17; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
sqlparse==0.3.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
statsd==3.3.0
stevedore==1.30.0
sqlparse==0.4.4 ; python_version >= '3.5'
statsd==4.0.1
stevedore==5.0.0 ; python_version >= '3.8'
tempita==0.5.2
termcolor==1.1.0
testrepository==0.0.20
testresources==2.0.1
testscenarios==0.5.0
testtools==2.4.0
traceback2==1.4.0
unittest2==1.1.0
urllib3==1.25.9; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'
uwsgi~=2.0.19.1
vine==1.3.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
wcwidth==0.2.4; sys_platform != 'win32'
webob==1.8.6; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
websocket-client==0.57.0
werkzeug==0.16.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
wrapt==1.12.1
yappi==1.2.5
zipp~=3.6.0
testtools==2.5.0 ; python_version >= '3.5'
tiddlyweb==2.4.3
tomli==2.0.1 ; python_version < '3.11'
tomlkit==0.11.8 ; python_version >= '3.7'
typing-extensions==3.7.2
urllib3==1.25.11 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
uwsgi==2.0.21
vine==1.3.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
virtualenv==20.23.0 ; python_version >= '3.7'
wcwidth==0.2.6
webob==1.8.7 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'
websocket-client==1.5.1 ; python_version >= '3.7'
werkzeug==2.1.2 ; python_version >= '3.7'
wheel==0.40.0 ; python_version >= '3.7'
wrapt==1.15.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
wsgi-intercept==1.11.0
xmltodict==0.13.0 ; python_version >= '3.4'
yappi==1.4.0
yq==3.2.1 ; python_version >= '3.6'
zipp==3.15.0

187
test-requirements.txt Normal file
View File

@ -0,0 +1,187 @@
alabaster==0.7.13
alembic==1.4.3
amqp==2.6.1
argcomplete==3.0.8
arrow==0.17.0
attrs==23.1.0
autopage==0.5.1
Babel==2.12.1
bandit==1.6.2
bcrypt==4.0.1
Beaker==1.12.0
cachetools==5.3.0
certifi==2022.12.7
cffi==1.15.1
chardet==3.0.4
charset-normalizer==3.1.0
click==6.7
click-default-group==1.2
cliff==3.10.1
cmd2==2.4.3
configparser==5.3.0
coverage==7.2.5
cryptography==3.4.8
debtcollector==2.5.0
Deckhand @ git+https://opendev.org/airship/deckhand.git@13c5199f18664d66ccdd3a1e54bb40ad2c293d1a
decorator==5.1.1
deepdiff==5.8.1
distlib==0.3.6
dnspython==2.3.0
docker==5.0.3
docutils==0.17
dogpile.cache==1.2.0
dulwich==0.21.3
eventlet==0.33.3
exceptiongroup==1.1.1
execnet==1.9.0
extras==1.0.0
falcon==3.1.1
fasteners==0.18
filelock==3.12.0
fixtures==3.0.0
flake8==3.8.4
flake8-import-order==0.18.2
futurist==2.4.1
gitdb==4.0.10
GitPython==3.1.31
google-auth==2.17.3
greenlet==2.0.2
hacking==4.1.0
html5lib==0.9999999
httpexceptor==1.4.0
idna==2.10
imagesize==1.4.1
importlib-metadata==2.1.3
iniconfig==2.0.0
iso8601==1.1.0
Jinja2==3.0.3
jsonpath-ng==1.5.3
jsonpath-rw==1.4.0
jsonpath-rw-ext==1.2.2
jsonpickle==1.4.1
jsonschema==3.2.0
keystoneauth1==5.1.1
keystonemiddleware==10.2.0
kombu==4.6.11
kubernetes==26.1.0
Mako==1.2.4
MarkupSafe==2.0.1
mccabe==0.6.1
mock==5.0.2
msgpack==1.0.5
netaddr==0.8.0
netifaces==0.11.0
networkx==3.1
nose==1.3.7
oauthlib==3.2.2
ordered-set==4.1.0
os-service-types==1.7.0
oslo.cache==2.10.1
oslo.concurrency==5.1.1
oslo.config==8.7.1
oslo.context==4.1.0
oslo.db==10.0.0
oslo.i18n==6.0.0
oslo.log==4.6.0
oslo.messaging==12.13.0
oslo.metrics==0.6.0
oslo.middleware==4.4.0
oslo.policy==3.10.1
oslo.serialization==4.2.0
oslo.service==3.1.1
oslo.utils==4.12.3
packaging==21.3
Paste==3.5.0
PasteDeploy==3.0.1
PasteScript==3.3.0
pbr==5.5.1
pip==23.0.1
pipenv==2023.2.4
platformdirs==3.5.0
pluggy==1.0.0
ply==3.11
prettytable==3.7.0
promenade @ git+https://opendev.org/airship/promenade.git@97e3721a03ca48d312891694972b827a53714bf9
prometheus-client==0.16.0
psycopg2-binary==2.9.6
PTable==0.9.2
pyasn1==0.5.0
pyasn1-modules==0.3.0
pycadf==3.1.1
pycodestyle==2.6.0
pycparser==2.21
pyflakes==2.2.0
Pygments==2.14.0
pylibyaml==0.1.0
pymongo==4.3.3
pyparsing==2.4.7
pyperclip==1.8.2
pyproject_api==1.5.0
pyrsistent==0.19.3
pytest==7.3.1
pytest-cov==4.0.0
pytest-xdist==3.2.1
python-barbicanclient==5.2.0
python-dateutil==2.8.1
python-editor==1.0.4
python-keystoneclient==5.1.0
python-memcached==1.59
python-mimeparse==1.6.0
python-subunit==1.4.0
pytz==2023.3
PyYAML==5.4.1
reno==4.0.0
repoze.lru==0.7
requests==2.23.0
requests-oauthlib==1.3.1
resolver==0.2.1
rfc3986==2.0.0
Routes==2.5.1
rsa==4.9
selector==0.10.1
setuptools==45.2.0
shipyard-client @ git+https://opendev.org/airship/shipyard.git@154a099b2843b1b42a7b8afa5fa98383cf3c1b7c#subdirectory=src/bin/shipyard_client
simplejson==3.19.1
six==1.16.0
smmap==5.0.0
snowballstemmer==2.2.0
Sphinx==3.3.1
sphinx-rtd-theme==0.5.0
sphinxcontrib-applehelp==1.0.4
sphinxcontrib-devhelp==1.0.2
sphinxcontrib-htmlhelp==2.0.1
sphinxcontrib-jsmath==1.0.1
sphinxcontrib-qthelp==1.0.3
sphinxcontrib-serializinghtml==1.1.5
SQLAlchemy==1.3.20
sqlalchemy-migrate==0.13.0
sqlparse==0.4.4
statsd==4.0.1
stevedore==5.0.0
Tempita==0.5.2
testfixtures==7.1.0
testrepository==0.0.20
testresources==2.0.1
testscenarios==0.5.0
testtools==2.5.0
tiddlyweb==2.4.3
tomli==2.0.1
tomlkit==0.11.8
typing-extensions==3.7.2
urllib3==1.25.11
uWSGI==2.0.21
vine==1.3.0
virtualenv==20.23.0
virtualenv-clone==0.5.7
wcwidth==0.2.6
WebOb==1.8.7
websocket-client==1.5.1
Werkzeug==2.1.2
wheel==0.40.0
wrapt==1.15.0
wsgi-intercept==1.11.0
xmltodict==0.13.0
yapf==0.28.0
yappi==1.4.0
yq==3.2.1
zipp==3.15.0

View File

@ -30,7 +30,7 @@ from tests.unit import test_utils
TEST_PARAMS = {
"site_name": "seaworthy",
"site_type": "foundry",
"repo_rev": '342c9eedd58f046a322ee2dd6752a9ec8fa992bb',
"repo_rev": '23e4bfc80aa6a3c845b31750d7593898f6e55e82',
"repo_name": "treasuremap",
"repo_url": "https://review.opendev.org/airship/treasuremap.git",
}
@ -756,7 +756,7 @@ class TestSiteSecretsActions(BaseCLIActionTest):
class TestTypeCliActions(BaseCLIActionTest):
"""Tests type-level CLI actions."""
def setup(self):
def setup_method(self, *args):
self.expected_types = ['foundry']
def _assert_table_has_expected_sites(self, table_output):
@ -795,7 +795,7 @@ class TestTypeCliActions(BaseCLIActionTest):
class TestSiteCliActionsWithSubdirectory(BaseCLIActionTest):
"""Tests site CLI actions with subdirectories in repository paths."""
def setup(self):
def setup_method(self, *args):
self.expected_sites = ['demo', 'gate-multinode', 'dev', 'dev-proxy']
def _assert_table_has_expected_sites(self, table_output):
@ -874,7 +874,7 @@ class TestCliSiteSubcommandsWithDecryptOption(BaseCLIActionTest):
"PEGLEG_PASSPHRASE": 'ytrr89erARAiPE34692iwUMvWqqBvC',
"PEGLEG_SALT": "MySecretSalt1234567890]["
})
def setup(self):
def setup_method(self, *args):
pegleg_main.run_config(
self.treasuremap_path, None, None, None, [], True, False)
pegleg_main.run_encrypt('zuul-tester', None, self.site_name)

View File

@ -25,7 +25,8 @@ from pegleg.engine.common import managed_document
CERT_HEADER = '-----BEGIN CERTIFICATE-----\n'
CERT_KEY_HEADER = '-----BEGIN RSA PRIVATE KEY-----\n'
PUBLIC_KEY_HEADER = '-----BEGIN PUBLIC KEY-----\n'
PRIVATE_KEY_HEADER = '-----BEGIN RSA PRIVATE KEY-----\n'
PRIVATE_KEY_HEADER_PKCS1 = '-----BEGIN RSA PRIVATE KEY-----\n'
PRIVATE_KEY_HEADER_PKCS8 = '-----BEGIN PRIVATE KEY-----\n'
PEGLEG_MANAGED_DOC_SCHEMA = 'pegleg/PeglegManagedDocument/v1'
CA_SCHEMA = 'deckhand/CertificateAuthority/v1'
@ -87,7 +88,8 @@ class TestPKIUtility(object):
assert isinstance(priv_key, dict), priv_key
assert PRIVATE_KEY_SCHEMA in priv_key['schema']
assert PRIVATE_KEY_HEADER in priv_key['data']
assert PRIVATE_KEY_HEADER_PKCS1 in priv_key['data'] or \
PRIVATE_KEY_HEADER_PKCS8 in priv_key['data']
def test_generate_certificate(self):
pki_obj = pki_utility.PKIUtility(duration=365)

View File

@ -32,7 +32,7 @@ For more information, see: https://storyboard.openstack.org/#!/story/2003762
class TestSelectableLinting(object):
def setup(self):
def setup_method(self, *args):
self.site_yaml_path = os.path.join(os.getcwd(), 'site_yamls')
def _exclude_all(self, except_code):

View File

@ -13,6 +13,13 @@
# limitations under the License.
- hosts: primary
roles:
- clear-firewall
- bindep
- ensure-docker
- ensure-python
- ensure-pip
tasks:
- include_vars: vars.yaml
@ -29,23 +36,14 @@
- debug:
msg: "{{ tags | to_json }}"
- name: Set ubuntu bionic base image
- name: Set ubuntu focal base image
set_fact:
base_image: "{{ base_image_bionic }}"
when: distro == 'ubuntu_bionic'
base_image: "{{ base_image_focal }}"
when: distro == 'ubuntu_focal'
- name: Set ubuntu xenial base image
set_fact:
base_image: "{{ base_image_xenial }}"
when: distro == 'ubuntu_xenial'
- name: Set ubuntu opensuse base image
set_fact:
base_image: "{{ base_image_opensuse }}"
when: distro == 'opensuse_15'
- name: Determine tags
shell: echo '{{ tags | to_json }}' | python {{ zuul.project.src_dir }}/tools/image_tags.py
shell: echo '{{ tags | to_json }}' | python3 {{ zuul.project.src_dir }}/tools/image_tags.py
environment:
BRANCH: "{{ zuul.branch | default('') }}"
CHANGE: "{{ zuul.change | default('') }}"
@ -61,50 +59,12 @@
shell: rm /etc/pip.conf
become: True
- name: Install Docker (Debian)
when: ansible_os_family == 'Debian'
- name: Install Docker python module for ansible docker login
block:
- file:
path: "{{ item }}"
state: directory
with_items:
- /etc/docker/
- /etc/systemd/system/docker.service.d/
- /var/lib/docker/
- mount:
path: /var/lib/docker/
src: tmpfs
fstype: tmpfs
opts: size=25g
state: mounted
- copy: "{{ item }}"
with_items:
- content: "{{ docker_daemon | to_json }}"
dest: /etc/docker/daemon.json
- src: files/docker-systemd.conf
dest: /etc/systemd/system/docker.service.d/
- apt_key:
url: https://download.docker.com/linux/ubuntu/gpg
- apt_repository:
repo: "deb http://{{ deb_docker_repo }} bionic stable"
- apt:
name: "{{ item }}"
allow_unauthenticated: True
with_items:
- docker-ce
- python3-pip
- python3-setuptools
- pip:
name: docker
version: 2.7.0
version: 4.4.4
executable: pip3
# NOTE(SamYaple): Allow all connections from containers to host so the
# containers can access the http server for git and wheels
- iptables:
action: insert
chain: INPUT
in_interface: docker0
jump: ACCEPT
become: True
- name: Make images

View File

@ -1,8 +0,0 @@
# NOTE(SamYaple): CentOS cannot be build with userns-remap enabled. httpd uses
# cap_set_file capability and there is no way to pass that in at build as of
# docker 17.06.
# TODO(SamYaple): Periodically check to see if this is possible in newer
# versions of Docker
[Service]
ExecStart=
ExecStart=/usr/bin/dockerd

View File

@ -10,7 +10,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
- hosts: ubuntu-bionic
- hosts: ubuntu-focal
tasks:
- name: lint-yaml
shell: |

17
tools/gate/run-cover-tests.sh Executable file
View File

@ -0,0 +1,17 @@
#!/usr/bin/env bash
set -e
posargs=$@
# cross-platform way to derive the number of logical cores
readonly num_cores=$(python -c 'import multiprocessing as mp; print(mp.cpu_count())')
if [ ${#posargs} -ge 1 ]; then
PATH=$PATH:~/.local/bin; pytest -vv -k ${posargs} -n $num_cores --cov=pegleg --cov-report \
html:cover --cov-report xml:cover/coverage.xml --cov-report term \
--cov-fail-under 87 tests/
else
pytest -n $num_cores
PATH=$PATH:~/.local/bin; pytest -vv -n $num_cores --cov=pegleg --cov-report \
html:cover --cov-report xml:cover/coverage.xml --cov-report term \
--cov-fail-under 87 tests/
fi
set +e

View File

@ -5,8 +5,8 @@ posargs=$@
# cross-platform way to derive the number of logical cores
readonly num_cores=$(python -c 'import multiprocessing as mp; print(mp.cpu_count())')
if [ ${#posargs} -ge 1 ]; then
pytest -k ${posargs} -n $num_cores
pytest -k ${posargs} -n $num_cores -vv
else
pytest -n $num_cores
pytest -n $num_cores -vv
fi
set +e

View File

@ -3,7 +3,7 @@
set -e
: ${WORKSPACE:=$(pwd)}
: ${IMAGE:=quay.io/airshipit/pegleg:latest-ubuntu_xenial}
: ${IMAGE:=quay.io/airshipit/pegleg:latest-ubuntu_focal}
: ${TERM_OPTS:=-it}

96
tox.ini
View File

@ -1,6 +1,6 @@
[tox]
envlist = py36,pep8,docs,cover
minversion = 2.3.1
envlist = py38,pep8,docs,cover
minversion = 2.38.0
skipsdist = True
[testenv]
@ -11,84 +11,75 @@ setenv =
LC_ALL=en_US.utf-8
PIPENV_VERBOSITY=-1
PIPENV_IGNORE_PIPFILE=1
deps =
pipenv
setuptools < 58.0.0
passenv = http_proxy https_proxy HTTP_PROXY HTTPS_PROXY no_proxy NO_PROXY PBR_VERSION
whitelist_externals =
deps=
-r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
passenv =
http_proxy
https_proxy
HTTP_PROXY
HTTPS_PROXY
no_proxy
NO_PROXY
PBR_VERSION
allowlist_externals =
bash
find
commands =
pipenv install --dev
find . -type f -name "*.pyc" -delete
bash -c "{toxinidir}/tools/install-cfssl.sh"
{toxinidir}/tools/gate/run-unit-tests.sh '{posargs}'
bash -c "{toxinidir}/tools/gate/run-unit-tests.sh '{posargs}'"
[testenv:fmt]
basepython = python3
deps =
pipenv
setuptools < 58.0.0
commands =
pipenv install --dev
yapf -ir {toxinidir}/pegleg {toxinidir}/tests
allowlist_externals =
yapf
[testenv:pep8]
basepython = python3
deps =
pipenv
setuptools < 58.0.0
safety
bandit
commands =
pipenv install --dev
command =
bash -c "{toxinidir}/tools/gate/whitespace-linter.sh"
bandit -r pegleg -n 5
bandit --skip B105 -r pegleg -n 5
flake8 {toxinidir}/pegleg
yapf -dr {toxinidir}/pegleg {toxinidir}/tests
whitelist_externals =
allowlist_externals =
bash
bandit
flake8
yapf
find
[testenv:docs]
basepython = python3
deps =
pipenv
setuptools < 58.0.0
-r{toxinidir}/doc/requirements.txt
deps=
-r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
-r{toxinidir}/doc/requirements.txt
commands =
pipenv install --dev
bash -c "{toxinidir}/tools/gate/build-docs.sh"
whitelist_externals =
allowlist_externals =
bash
[testenv:bandit]
basepython = python3
deps =
bandit
commands = bandit -r pegleg -n 5
commands = bandit --skip B105 -r pegleg -n 5
[testenv:safety]
basepython = python3
deps =
pipenv
setuptools < 58.0.0
safety
commands =
pipenv check
safety check -r {toxinidir}/requirements.txt -r {toxinidir}/doc/requirements.txt --full-report
[testenv:cover]
basepython = python3
deps =
pipenv
setuptools < 58.0.0
commands =
pipenv install --dev
{toxinidir}/tools/install-cfssl.sh
bash -c 'PATH=$PATH:~/.local/bin; pytest --cov=pegleg --cov-report \
html:cover --cov-report xml:cover/coverage.xml --cov-report term \
--cov-fail-under 87 tests/'
whitelist_externals =
bash -c {toxinidir}/tools/install-cfssl.sh
bash -c {toxinidir}/tools/gate/run-cover-tests.sh
allowlist_externals =
bash
[testenv:update-requirements]
@ -96,18 +87,27 @@ deps =
pipenv
setuptools < 58.0.0
commands =
rm -f Pipfile.lock
rm -f requirements.txt
rm -f test-requirements.txt
pipenv lock --clear
bash -c "pipenv lock -r > {toxinidir}/requirements.txt"
whitelist_externals =
bash
sh -c "PIPENV_VERBOSITY=-1 pipenv requirements > requirements.txt"
pipenv install --dev
sh -c "pip freeze --all | grep -vE 'pegleg|pyinotify|pkg-resources==0.0.0' > test-requirements.txt"
allowlist_externals =
rm
sh
[testenv:releasenotes]
basepython = python3
deps = -r{toxinidir}/doc/requirements.txt
deps=
-r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
-r{toxinidir}/doc/requirements.txt
commands =
rm -rf releasenotes/build
sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html
whitelist_externals =
allowlist_externals =
rm
[testenv:venv]

View File

@ -1,3 +1,3 @@
DECKHAND_VERSION=commit/5cd799cc5d04527ac782270008ff647b3779ff05
SHIPYARD_VERSION=commit/e3e71f7d1632c0dc13b436a32c238f9da76d6d88
PROMENADE_VERSION=commit/c10165c144e0a18137596a3c89f1339d6ed30d0c
DECKHAND_VERSION=commit/03f6932e16aa0b72a32a10fc04a52125c45dd5d7
SHIPYARD_VERSION=commit/154a099b2843b1b42a7b8afa5fa98383cf3c1b7c
PROMENADE_VERSION=commit/97e3721a03ca48d312891694972b827a53714bf9