openstack
/
tripleo-ci
Code Issues Proposed changes

Add content provider using current tag to build containers for minor update 

Add content provider job, which builds containers using `current` dlrn
tag on same release branch as regular content-provider job.

Apart from new content provider job, the change adds new variable called
`minor_update`, which enables to consume containers created by new
content provider job in depended multinode-oooq-container-updates job.

minor_update variable is used in featureset068.
The featureset is activated in jobs for wallaby branch and onwards.
Train job uses featureset038 as previously.

Related-Bug: #2015993
Change-Id: I7a7713be0cd15321d5b16f6b8ca85f06d296219f
This commit is contained in:
Sofer Athlan-Guyot 2023-04-12 17:23:29 +02:00 committed by mciecier
parent bd726e63d5
commit 9e7cf30337
11 changed files with 188 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
playbooks/tripleo-ci/run-provider.yml
View File

@ -19,6 +19,11 @@
name: build-containers
tasks_from: set_ci_branch.yml
- name: Combine override_repos with ci_branch variable
set_fact:
ci_branch: "{{ ci_branch + '-' + override_repos }}"
when: override_repos != ''
- name: Set branch variables
set_fact:
provider_job_branch: "{{ ci_branch }}"

4
roles/build-containers/tasks/set_ci_branch.yml
View File

@ -19,6 +19,10 @@
ci_branch: "{{ osp_branch }}"
when: osp_branch is defined
- name: Set override_repos for buildcontainers_override_repos
set_fact:
override_repos: "{{ buildcontainers_override_repos | default('') }}"
- name: Fail if we did not set ci_branch fact
fail:
msg: >-

1
roles/run-test/tasks/main.yaml
View File

@ -15,6 +15,7 @@
- 'dlrn_hash_tag_newest'
- 'extra_args'
- 'enable_fips'
- 'promote_source'
- 'run_tempest'
- 'standalone_container_cli'
- 'standalone_environment_files'

12
roles/run-test/templates/toci_gate_test.sh.j2
View File

@ -105,8 +105,16 @@ elif is_featureset undercloud_upgrade "{{ job_featureset_file }}"; then
{% if provider_dlrn_hash_tag_branch is defined %}
EMIT_RELEASES_EXTRA_ARGS="$EMIT_RELEASES_EXTRA_ARGS --content-provider-hashes={{ provider_dlrn_hash_tag_branch.keys()|zip(provider_dlrn_hash_tag_branch.values())|map('join',':')|join(';') }} --target-branch-override=$UPGRADE_RELEASE --install-branch-override=$QUICKSTART_RELEASE"
{% endif %}
# For minor_update we stay on the same branch, but we need to pass zuul information of 'current' content provider job in target-branch-override.
elif is_featureset minor_update "{{ job_featureset_file }}"; then
echo "minor update centos9/wallaby"
{% if provider_dlrn_hash_tag_branch is defined %}
EMIT_RELEASES_EXTRA_ARGS="$EMIT_RELEASES_EXTRA_ARGS --content-provider-hashes={{ provider_dlrn_hash_tag_branch.keys()|zip(provider_dlrn_hash_tag_branch.values())|map('join',':')|join(';') }} --target-branch-override=${QUICKSTART_RELEASE}-current --install-branch-override=$QUICKSTART_RELEASE"
{% endif %}
# This is a case for train branch, which uses featureset038. In featureset038 we test update workflow itself, no containers or rpms update happens.
# Target branch is overridden to be exactly the same as for deployment, because only containers used during deployment are in local container registry.
elif is_featureset overcloud_update "{{ job_featureset_file }}"; then
echo ""
echo "minor update centos8/train"
{% if provider_dlrn_hash_tag_branch is defined %}
EMIT_RELEASES_EXTRA_ARGS="$EMIT_RELEASES_EXTRA_ARGS --content-provider-hashes={{ provider_dlrn_hash_tag_branch.keys()|zip(provider_dlrn_hash_tag_branch.values())|map('join',':')|join(';') }} --target-branch-override=$QUICKSTART_RELEASE"
{% endif %}
@ -124,7 +132,7 @@ QUICKSTART_RELEASE="promotion-testing-hash-${QUICKSTART_RELEASE}"
EMIT_RELEASES_EXTRA_ARGS="$EMIT_RELEASES_EXTRA_ARGS --is-periodic"
{% endif %}
if [[ -f "$RELEASES_SCRIPT" ]] && [[ {{ featureset }} =~ 037|047|050|056|065|067 ]]; then
if [[ -f "$RELEASES_SCRIPT" ]] && [[ {{ featureset }} =~ 037|047|050|056|065|067|068 ]]; then
if [[ -f "$RELEASES_FILE_OUTPUT" ]]; then
echo "Importing ready releases.sh file - skipping releases.sh file generation."

13
roles/run-test/templates/toci_quickstart.sh.j2
View File

@ -83,7 +83,7 @@ if [[ -f "$RELEASES_FILE_OUTPUT" ]]; then
["multinode-overcloud.yml"]=$(get_extra_vars_from_release \
$OVERCLOUD_DEPLOY_RELEASE $OVERCLOUD_DEPLOY_HASH)
["multinode-overcloud-update.yml"]=$(get_extra_vars_from_release \
$OVERCLOUD_DEPLOY_RELEASE $OVERCLOUD_DEPLOY_HASH)
$OVERCLOUD_TARGET_RELEASE $OVERCLOUD_TARGET_HASH)
["multinode-overcloud-upgrade.yml"]=$(get_extra_vars_from_release \
$OVERCLOUD_TARGET_RELEASE $OVERCLOUD_TARGET_HASH)
["multinode-validate.yml"]=$(get_extra_vars_from_release \
@ -114,6 +114,17 @@ declare -A PLAYBOOKS_ARGS=(
["multinode.yml"]=" --extra-vars validation_args='--validation-errors-nonfatal' "
)
# minor_update variable is set to true only in featureset068
# We need to override container_registry_release to get ip of docker registry and dlrn hash from content provider which provides containers with 'current' tag
# Containers are required to be pulled both to undercloud and overcloud to run minor update on the same branch
# Also to trigger update of undercloud and overcloud from correct playbook, overcloud_update and undercloud_update extra-vars are set to false respectively
if is_featureset minor_update "{{ job_featureset_file }}"; then
PLAYBOOKS_ARGS=(
["multinode-undercloud-upgrade.yml"]=" --extra-vars container_registry_release='${QUICKSTART_RELEASE}-current' --extra-vars promote_source='current' --extra-vars overcloud_update='false' "
["multinode-overcloud-update.yml"]=" --extra-vars container_registry_release='${QUICKSTART_RELEASE}-current' --extra-vars promote_source='current' --extra-vars undercloud_update='false' "
)
fi
mkdir -p $LOCAL_WORKING_DIR
# TODO(gcerami) parametrize hosts
cp {{ tripleo_root }}/tripleo-ci/toci-quickstart/config/testenv/{{ environment_infra }}_hosts $LOCAL_WORKING_DIR/hosts

27
scripts/emit_releases_file/emit_releases_file.py
View File

@ -234,7 +234,7 @@ def compose_releases_dictionary(
):
current_hash = content_provider_hashes[target_branch_override]
logger.info(
"Using hash override {} for branch {}".format(
"Using hash override {} from content provider hashes map for branch {}".format(
current_hash, target_branch_override
)
)
@ -325,6 +325,31 @@ def compose_releases_dictionary(
releases_dictionary['standalone_deploy_newest_hash'] = install_newest_hash
releases_dictionary['standalone_deploy_hash'] = install_hash
elif featureset.get('minor_update'):
if is_periodic:
previous_hash = get_dlrn_hash(
stable_release, PREVIOUS_HASH_NAME, distro_name, distro_version
)
releases_dictionary['overcloud_deploy_hash'] = previous_hash
else:
target_newest_hash = get_dlrn_hash(
stable_release, NEWEST_HASH_NAME, distro_name, distro_version
)
releases_dictionary['undercloud_target_hash'] = target_newest_hash
releases_dictionary['overcloud_target_hash'] = target_newest_hash
if content_provider_hashes is not None and content_provider_hashes.get(
install_branch_override
):
install_hash = content_provider_hashes[install_branch_override]
releases_dictionary['undercloud_install_hash'] = install_hash
releases_dictionary['overcloud_deploy_hash'] = install_hash
if content_provider_hashes is not None and content_provider_hashes.get(
target_branch_override
):
current_hash = content_provider_hashes[target_branch_override]
releases_dictionary['undercloud_target_hash'] = current_hash
releases_dictionary['overcloud_target_hash'] = current_hash
elif featureset.get('overcloud_update'):
logger.info('Doing an overcloud update')
previous_hash = get_dlrn_hash(

149
scripts/emit_releases_file/test_release_name.py
View File

@ -606,82 +606,6 @@ def test_period_standalone_upgrade_is_n_minus_one_to_n(
@pytest.mark.parametrize(
'stable_release,expected_releases',
[
(
'master',
{
'undercloud_install_release': 'master',
'undercloud_install_hash': 'current-tripleo',
'undercloud_target_release': 'master',
'undercloud_target_hash': 'current-tripleo',
'overcloud_deploy_release': 'master',
'overcloud_deploy_hash': 'previous-current-tripleo',
'overcloud_target_release': 'master',
'overcloud_target_hash': 'current-tripleo',
'standalone_deploy_newest_hash': 'current',
'standalone_deploy_hash': 'current-tripleo',
'standalone_deploy_release': 'master',
'standalone_target_newest_hash': 'current',
'standalone_target_hash': 'current-tripleo',
'standalone_target_release': 'master',
},
),
(
'wallaby',
{
'undercloud_install_release': 'wallaby',
'undercloud_install_hash': 'current-tripleo',
'undercloud_target_release': 'wallaby',
'undercloud_target_hash': 'current-tripleo',
'overcloud_deploy_release': 'wallaby',
'overcloud_deploy_hash': 'previous-current-tripleo',
'overcloud_target_release': 'wallaby',
'overcloud_target_hash': 'current-tripleo',
'standalone_deploy_release': 'wallaby',
'standalone_deploy_newest_hash': 'current',
'standalone_deploy_hash': 'current-tripleo',
'standalone_target_release': 'wallaby',
'standalone_target_newest_hash': 'current',
'standalone_target_hash': 'current-tripleo',
},
),
(
'victoria',
{
'undercloud_install_release': 'victoria',
'undercloud_install_hash': 'current-tripleo',
'undercloud_target_release': 'victoria',
'undercloud_target_hash': 'current-tripleo',
'overcloud_deploy_release': 'victoria',
'overcloud_deploy_hash': 'previous-current-tripleo',
'overcloud_target_release': 'victoria',
'overcloud_target_hash': 'current-tripleo',
'standalone_deploy_release': 'victoria',
'standalone_deploy_newest_hash': 'current',
'standalone_deploy_hash': 'current-tripleo',
'standalone_target_release': 'victoria',
'standalone_target_newest_hash': 'current',
'standalone_target_hash': 'current-tripleo',
},
),
(
'ussuri',
{
'undercloud_install_release': 'ussuri',
'undercloud_install_hash': 'current-tripleo',
'undercloud_target_release': 'ussuri',
'undercloud_target_hash': 'current-tripleo',
'overcloud_deploy_release': 'ussuri',
'overcloud_deploy_hash': 'previous-current-tripleo',
'overcloud_target_release': 'ussuri',
'overcloud_target_hash': 'current-tripleo',
'standalone_deploy_release': 'ussuri',
'standalone_deploy_newest_hash': 'current',
'standalone_deploy_hash': 'current-tripleo',
'standalone_target_release': 'ussuri',
'standalone_target_newest_hash': 'current',
'standalone_target_hash': 'current-tripleo',
},
),
(
'train',
{
@ -703,7 +627,7 @@ def test_period_standalone_upgrade_is_n_minus_one_to_n(
),
],
)
def test_overcloud_update_target_is_hash(
def test_overcloud_update_train_target_is_hash(
hash_mock, hash_mock_setup, stable_release, expected_releases
):
expected_release = expected_releases['overcloud_deploy_release']
@ -728,6 +652,75 @@ def test_overcloud_update_target_is_hash(
)
@mock.patch('emit_releases_file.get_dlrn_hash')
@pytest.mark.parametrize(
'stable_release,expected_releases',
[
(
'master',
{
'undercloud_install_release': 'master',
'undercloud_install_hash': 'current-tripleo',
'undercloud_target_release': 'master',
'undercloud_target_hash': 'current',
'overcloud_deploy_release': 'master',
'overcloud_deploy_hash': 'current-tripleo',
'overcloud_target_release': 'master',
'overcloud_target_hash': 'current',
'standalone_deploy_newest_hash': 'current',
'standalone_deploy_hash': 'current-tripleo',
'standalone_deploy_release': 'master',
'standalone_target_newest_hash': 'current',
'standalone_target_hash': 'current-tripleo',
'standalone_target_release': 'master',
},
),
(
'wallaby',
{
'undercloud_install_release': 'wallaby',
'undercloud_install_hash': 'current-tripleo',
'undercloud_target_release': 'wallaby',
'undercloud_target_hash': 'current',
'overcloud_deploy_release': 'wallaby',
'overcloud_deploy_hash': 'current-tripleo',
'overcloud_target_release': 'wallaby',
'overcloud_target_hash': 'current',
'standalone_deploy_release': 'wallaby',
'standalone_deploy_newest_hash': 'current',
'standalone_deploy_hash': 'current-tripleo',
'standalone_target_release': 'wallaby',
'standalone_target_newest_hash': 'current',
'standalone_target_hash': 'current-tripleo',
},
),
],
)
def test_overcloud_minor_update_target_is_hash(
hash_mock, hash_mock_setup, stable_release, expected_releases
):
expected_release = expected_releases['overcloud_deploy_release']
hash_mock_setup(
hash_mock,
{
(stable_release, 'current-tripleo'): 'current-tripleo',
(stable_release, 'current'): 'current',
(expected_release, 'current'): 'current',
},
)
featureset = {
'minor_update': True,
}
upgrade_from = False
assert (
compose_releases_dictionary(stable_release, featureset, upgrade_from)
== expected_releases
)
@mock.patch('emit_releases_file.get_dlrn_hash')
@pytest.mark.parametrize(
'stable_release,expected_releases',
@ -829,7 +822,7 @@ def test_overcloud_update_target_is_hash(
),
],
)
def test_period_overcloud_update_target_is_hash(
def test_periodic_overcloud_update_target_is_hash(
hash_mock, hash_mock_setup, stable_release, expected_releases
):
expected_release = expected_releases['overcloud_deploy_release']

20
zuul.d/content-provider.yaml
View File

@ -32,6 +32,15 @@
vars:
containers_base_image: registry.access.redhat.com/ubi9:latest
- job:
name: tripleo-ci-centos-9-content-provider-current
parent: tripleo-ci-centos-9-content-provider
vars:
buildcontainers_override_repos: current
featureset_override:
dlrn_hash_tag: current
promote_source: current
- job:
name: tripleo-ci-centos-9-content-provider-wallaby
parent: tripleo-ci-centos-9-content-provider
@ -39,6 +48,17 @@
vars:
branch_override: stable/wallaby
- job:
name: tripleo-ci-centos-9-content-provider-current-wallaby
parent: tripleo-ci-centos-9-content-provider-current
override-checkout: stable/wallaby
vars:
branch_override: stable/wallaby
buildcontainers_override_repos: current
featureset_override:
dlrn_hash_tag: current
promote_source: current
- job:
name: tripleo-ci-centos-9-content-provider-zed
parent: tripleo-ci-centos-9-content-provider

1
zuul.d/multinode-jobs.yaml
View File

@ -120,6 +120,7 @@
# upgrade/update featuresets
- ^config/general_config/featureset037.yml
- ^config/general_config/featureset051.yml
- ^config/general_config/featureset068.yml
- zuul.d/upgrades-jobs-templates.yaml
- zuul.d/build-containers.yaml
- zuul.d/build-image.yaml

26
zuul.d/periodic.yaml
View File

@ -4,7 +4,7 @@
periodic:
jobs:
- tripleo-ci-centos-9-content-provider:
irrelevant-files:
irrelevant-files: &unbranched_content_provider_files
- .*molecule.*
- ^.*\.md$
- ^.*\.rst$
@ -21,8 +21,17 @@
- ^setup.*$
vars:
force_non_periodic: true
- tripleo-ci-centos-9-containers-multinode: &unbranched_vars_periodic
- tripleo-ci-centos-9-content-provider-current:
irrelevant-files: *unbranched_content_provider_files
vars:
force_non_periodic: true
buildcontainers_override_repos: current
featureset_override:
dlrn_hash_tag: current
promote_source: current
- tripleo-ci-centos-9-containers-multinode:
vars: &unbranched_vars_periodic
force_non_periodic: true
consumer_job: true
build_container_images: false
@ -30,8 +39,15 @@
- build
dependencies:
- tripleo-ci-centos-9-content-provider
- tripleo-ci-centos-9-standalone: *unbranched_vars_periodic
- tripleo-ci-centos-9-scenario000-multinode-oooq-container-updates: *unbranched_vars_periodic
- tripleo-ci-centos-9-standalone:
vars: *unbranched_vars_periodic
dependencies:
- tripleo-ci-centos-9-content-provider
- tripleo-ci-centos-9-scenario000-multinode-oooq-container-updates:
vars: *unbranched_vars_periodic
dependencies:
- tripleo-ci-centos-9-content-provider
- tripleo-ci-centos-9-content-provider-current
- project-template:
name: tripleo-multinode-branchful
@ -161,11 +177,13 @@
- tripleo-ci-centos-9-content-provider-wallaby: *force_non_periodic
- tripleo-ci-centos-8-content-provider-wallaby: *force_non_periodic
- tripleo-ci-centos-8-content-provider-train: *force_non_periodic
- tripleo-ci-centos-9-content-provider-current-wallaby: *force_non_periodic
- tripleo-ci-centos-9-scenario000-multinode-oooq-container-updates:
irrelevant-files: *multinode_ignored
vars: *consumer_vars
dependencies:
- tripleo-ci-centos-9-content-provider-wallaby
- tripleo-ci-centos-9-content-provider-current-wallaby
- tripleo-ci-centos-8-undercloud-ffu-wallaby:
vars: *consumer_vars
irrelevant-files: *undercloud_ignored

18
zuul.d/upgrades-jobs-templates.yaml
View File

@ -7,6 +7,7 @@
check:
jobs:
- tripleo-ci-centos-9-content-provider
- tripleo-ci-centos-9-content-provider-current
- tripleo-ci-centos-9-content-provider-zed
- tripleo-ci-centos-9-scenario000-multinode-oooq-container-updates: &cs9_update_all
irrelevant-files: &multinode_ignored
@ -54,6 +55,7 @@
- build
dependencies:
- tripleo-ci-centos-9-content-provider
- tripleo-ci-centos-9-content-provider-current
- tripleo-ci-centos-9-undercloud-upgrade: &cs9_undercloud_upgrade
vars: *consumer_vars
irrelevant-files: &undercloud_ignored
@ -101,6 +103,7 @@
# upgrade/update featuresets
- ^config/general_config/featureset037.yml
- ^config/general_config/featureset051.yml
- ^config/general_config/featureset068.yml
# unit tests
- ^tripleoclient/tests/.*$
- ^tripleo_common/tests/.*$
@ -112,6 +115,7 @@
gate:
jobs:
- tripleo-ci-centos-9-content-provider
- tripleo-ci-centos-9-content-provider-current
- tripleo-ci-centos-9-content-provider-zed
- tripleo-ci-centos-9-undercloud-upgrade: *cs9_undercloud_upgrade
- tripleo-ci-centos-9-scenario000-multinode-oooq-container-updates: *cs9_update_all
@ -143,6 +147,7 @@
check:
jobs:
- tripleo-ci-centos-9-content-provider
- tripleo-ci-centos-9-content-provider-current
- tripleo-ci-centos-9-scenario000-multinode-oooq-container-updates: *cs9_update_all
- tripleo-ci-centos-8-content-provider
- tripleo-ci-centos-8-content-provider-train
@ -161,6 +166,7 @@
jobs:
- tripleo-ci-centos-9-content-provider
- tripleo-ci-centos-8-content-provider
- tripleo-ci-centos-9-content-provider-current
- tripleo-ci-centos-8-content-provider-train
- tripleo-ci-centos-9-scenario000-multinode-oooq-container-updates: *cs9_update_all
- tripleo-ci-centos-8-undercloud-ffu-wallaby: *undercloud_ffu
@ -210,15 +216,23 @@
parent: tripleo-ci-base-multinode-centos-9
voting: true
vars:
playbooks:
- quickstart.yml
- multinode-undercloud.yml
- multinode-overcloud-prep.yml
- multinode-overcloud.yml
- multinode-undercloud-upgrade.yml
- multinode-overcloud-update.yml
- multinode-validate.yml
nodes: 1ctlr
featureset: '037'
featureset: '068'
# Note (dmsimard): This enforces a minimum TTL for DNS records, see
# https://review.opendev.org/#/c/523178/ for context
unbound_cache_min_ttl: 900
extra_tags:
- undercloud-upgrade
- overcloud-update
#
# Undercloud upgrade jobs