Refresh local registry auth info each time when access local registry

Local registry uses admin account password as authentication info.
And this password may be changed by openstack client at any time.
When try to download images from local registry, auth info cannot
be cached, otherwise it may lead to authentication failure in keystone,
and account be locked at the end.
For this specific case, there is host-swact first, then function
"_upgrade_downgrade_kube_networking" in sysinv conductor is called.
And upgrade-k8s-networking.yml is executed which will try to download
kube network images from local registry. During this period, admin
account password is changed. And lead to account be locked due to
authentication failure in keystone.
With this update, there is still possibility that password be changed
just after get operation. And due to the images download are run in
parallel with multi threads, so account lock may still hit. This
change could minimize the issue rate, but cannot fix all.

Closes-Bug: 1853017

Change-Id: I686616937031a3f7ac6d65e5b118511dc549ab85
Signed-off-by: Shuicheng Lin <shuicheng.lin@intel.com>

playbookconfig/src/playbooks/roles/common/push-docker-images/files/download_images.py

@@ -28,6 +28,13 @@ DEFAULT_REGISTRIES = {
 registries = json.loads(os.environ['REGISTRIES'])
 
 
+def get_local_registry_auth():
+    password = str(keyring.get_password("CGCS", "admin"))
+    if not password:
+        raise Exception("Local registry password not found.")
+    return dict(username="admin", password=password)
+
+
 def download_an_image(img):
     # This function is to pull image from public/private
     # registry and push to local registry.
@@ -56,23 +63,25 @@ def download_an_image(img):
     local_img = 'registry.local:9001/' + new_img
     err_msg = " Image download failed: %s" % target_img
 
-    password = str(keyring.get_password("CGCS", "admin"))
-    if not password:
-        raise Exception("Local registry password not found.")
-    auth = '{0}:{1}'.format('admin', password)
-
     for i in range(MAX_DOWNLOAD_ATTEMPTS):
         try:
             client = docker.APIClient()
             client.pull(target_img)
             print("Image download succeeded: %s" % target_img)
             client.tag(target_img, local_img)
-            client.push(local_img)
+            # admin password may be changed by openstack client in parallel.
+            # So we cannot cache auth info, need refresh it each time.
+            auth = get_local_registry_auth()
+            client.push(local_img, auth_config=auth)
             print("Image push succeeded: %s" % local_img)
             # due to crictl doesn't support push function, docker client is used
             # to pull and push image to local registry, then crictl download image
             # from local registry.
-            subprocess.check_call(["crictl", "pull", "--creds", auth, local_img])
+            # admin password may be changed by openstack client in parallel.
+            # So we cannot cache auth info, need refresh it each time.
+            auth = get_local_registry_auth()
+            auth_str = '{0}:{1}'.format(auth['username'], auth['password'])
+            subprocess.check_call(["crictl", "pull", "--creds", auth_str, local_img])
             print("Image %s download succeeded by containerd" % target_img)
             # except armada/tiller, other docker images could be removed.
             # TODO: run armada with containerd.

playbookconfig/src/playbooks/roles/common/push-docker-images/tasks/main.yml

@@ -133,12 +133,6 @@
 - set_fact:
     local_registry_credentials: "{{ local_registry_credentials_output.stdout }}"
 
-- name: Log in to local registry
-  docker_login:
-    registry: "{{ local_registry }}"
-    username: "{{ local_registry_credentials['username'] }}"
-    password: "{{ local_registry_credentials['password'] }}"
-
 - name: Download images and push to local registry
   script: download_images.py {{ download_images }}
   register: download_images_output
@@ -160,8 +154,3 @@
     - "{{ docker_registry }}"
   when: item.username is defined
   no_log: true
-
-- name: Log out of local registry
-  docker_login:
-    registry: "{{ local_registry }}"
-    state: absent