From d6cff0496dcf52655eba340e1e57b1d973040edf Mon Sep 17 00:00:00 2001 From: Shuicheng Lin Date: Thu, 12 Mar 2020 14:34:09 +0800 Subject: [PATCH] Refresh local registry auth info each time when access local registry Local registry uses admin account password as authentication info. And this password may be changed by openstack client at any time. When try to download images from local registry, auth info cannot be cached, otherwise it may lead to authentication failure in keystone, and account be locked at the end. For this specific case, there is host-swact first, then function "_upgrade_downgrade_kube_networking" in sysinv conductor is called. And upgrade-k8s-networking.yml is executed which will try to download kube network images from local registry. During this period, admin account password is changed. And lead to account be locked due to authentication failure in keystone. With this update, there is still possibility that password be changed just after get operation. And due to the images download are run in parallel with multi threads, so account lock may still hit. This change could minimize the issue rate, but cannot fix all. Closes-Bug: 1853017 Change-Id: I686616937031a3f7ac6d65e5b118511dc549ab85 Signed-off-by: Shuicheng Lin --- .../files/download_images.py | 23 +++++++++++++------ .../common/push-docker-images/tasks/main.yml | 11 --------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/playbookconfig/src/playbooks/roles/common/push-docker-images/files/download_images.py b/playbookconfig/src/playbooks/roles/common/push-docker-images/files/download_images.py index bbdecef1f..ad6547d28 100644 --- a/playbookconfig/src/playbooks/roles/common/push-docker-images/files/download_images.py +++ b/playbookconfig/src/playbooks/roles/common/push-docker-images/files/download_images.py @@ -28,6 +28,13 @@ DEFAULT_REGISTRIES = { registries = json.loads(os.environ['REGISTRIES']) +def get_local_registry_auth(): + password = str(keyring.get_password("CGCS", "admin")) + if not password: + raise Exception("Local registry password not found.") + return dict(username="admin", password=password) + + def download_an_image(img): # This function is to pull image from public/private # registry and push to local registry. @@ -56,23 +63,25 @@ def download_an_image(img): local_img = 'registry.local:9001/' + new_img err_msg = " Image download failed: %s" % target_img - password = str(keyring.get_password("CGCS", "admin")) - if not password: - raise Exception("Local registry password not found.") - auth = '{0}:{1}'.format('admin', password) - for i in range(MAX_DOWNLOAD_ATTEMPTS): try: client = docker.APIClient() client.pull(target_img) print("Image download succeeded: %s" % target_img) client.tag(target_img, local_img) - client.push(local_img) + # admin password may be changed by openstack client in parallel. + # So we cannot cache auth info, need refresh it each time. + auth = get_local_registry_auth() + client.push(local_img, auth_config=auth) print("Image push succeeded: %s" % local_img) # due to crictl doesn't support push function, docker client is used # to pull and push image to local registry, then crictl download image # from local registry. - subprocess.check_call(["crictl", "pull", "--creds", auth, local_img]) + # admin password may be changed by openstack client in parallel. + # So we cannot cache auth info, need refresh it each time. + auth = get_local_registry_auth() + auth_str = '{0}:{1}'.format(auth['username'], auth['password']) + subprocess.check_call(["crictl", "pull", "--creds", auth_str, local_img]) print("Image %s download succeeded by containerd" % target_img) # except armada/tiller, other docker images could be removed. # TODO: run armada with containerd. diff --git a/playbookconfig/src/playbooks/roles/common/push-docker-images/tasks/main.yml b/playbookconfig/src/playbooks/roles/common/push-docker-images/tasks/main.yml index ecceed2ef..a2b282b9c 100644 --- a/playbookconfig/src/playbooks/roles/common/push-docker-images/tasks/main.yml +++ b/playbookconfig/src/playbooks/roles/common/push-docker-images/tasks/main.yml @@ -133,12 +133,6 @@ - set_fact: local_registry_credentials: "{{ local_registry_credentials_output.stdout }}" -- name: Log in to local registry - docker_login: - registry: "{{ local_registry }}" - username: "{{ local_registry_credentials['username'] }}" - password: "{{ local_registry_credentials['password'] }}" - - name: Download images and push to local registry script: download_images.py {{ download_images }} register: download_images_output @@ -160,8 +154,3 @@ - "{{ docker_registry }}" when: item.username is defined no_log: true - -- name: Log out of local registry - docker_login: - registry: "{{ local_registry }}" - state: absent