patch of 446820 avoid repeat get registry image

Change-Id: I218e835376b25ab8c2e4702cf827351a52028fd2
Signed-off-by: Yao Lu <lu.yao135@zte.com.cn>

code/daisy/daisy/api/backends/kolla/api.py

@@ -51,7 +51,8 @@ class API(driver.DeploymentDriver):
         kolla_install_task = instl.KOLLAInstallTask(req, cluster_id)
         kolla_install_task.start()
 
-    def upgrade(self, req, cluster_id, version_id, vpatch_id, update_file, hosts):
+    def upgrade(self, req, cluster_id, version_id, vpatch_id,
+                update_file, hosts):
         """
         update openstack to a cluster.
 
@@ -62,7 +63,8 @@ class API(driver.DeploymentDriver):
         LOG.info(
             _("Begin to update OpenStack nodes, please waiting...."))
 
-        kolla_install_task = upgrd.KOLLAUpgradeTask(req, cluster_id, version_id, update_file)
+        kolla_install_task = upgrd.KOLLAUpgradeTask(req, cluster_id,
+                                                    version_id, update_file)
         kolla_install_task.start()
 
     def update_progress_to_db(self, req, update_info, discover_host_meta):

code/daisy/daisy/api/backends/kolla/common.py

@@ -287,10 +287,10 @@ def check_and_get_kolla_version(daisy_kolla_pkg_path, file_name=None):
     kolla_version_pkg_file = ""
     if file_name:
         get_kolla_version_pkg = "ls %s| grep %s$" % (daisy_kolla_pkg_path,
-                                                    file_name)
+                                                     file_name)
     else:
         get_kolla_version_pkg = "ls %s| grep ^kolla.*\.tgz$"\
-                               % daisy_kolla_pkg_path
+                                % daisy_kolla_pkg_path
     obj = subprocess.Popen(get_kolla_version_pkg,
                            shell=True,
                            stdout=subprocess.PIPE,
@@ -304,19 +304,19 @@ def check_and_get_kolla_version(daisy_kolla_pkg_path, file_name=None):
     return kolla_version_pkg_file
 
 
-def version_load(kolla_version_pkg_file, fp):
+def version_load(kolla_version_pkg_file):
-    tar_for_kolla_version = 'tar mzxvf %s' % kolla_version_pkg_file
+    tar_for_kolla_version = 'cd %s && tar mzxvf %s' % (daisy_kolla_ver_path,
+                                                       kolla_version_pkg_file)
     subprocess.call(tar_for_kolla_version, shell=True)
     get_container_id = "docker ps -a |grep registry |awk -F ' ' '{printf $1}' "
     container_id = subprocess.check_output(get_container_id, shell=True)
     if container_id:
         stop_container = 'docker stop %s' % container_id
-        daisy_cmn.subprocess_call(stop_container, fp)
+        daisy_cmn.subprocess_call(stop_container)
         remove_container = 'docker rm %s' % container_id
-        daisy_cmn.subprocess_call(remove_container, fp)
+        daisy_cmn.subprocess_call(remove_container)
     registry_file = daisy_kolla_ver_path + "/tmp/registry"
     daisy_cmn.subprocess_call(
         'docker run -d -p 4000:5000 --restart=always \
         -e REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY=/tmp/registry \
-        -v %s:/tmp/registry  --name registry registry:2' % registry_file, fp)
+        -v %s:/tmp/registry  --name registry registry:2' % registry_file)
-

code/daisy/daisy/api/backends/kolla/install.py

@@ -176,7 +176,7 @@ def get_cluster_kolla_config(req, cluster_id):
     host_name_ip = {}
     host_name_ip_list = []
     version_flag = False
-    version_path = kolla_file + '/docker/'
+    version_path = kolla_cmn.daisy_kolla_ver_path
     for parent, dirnames, filenames in os.walk(version_path):
         for filename in filenames:
             if filename.endswith('.version'):
@@ -448,20 +448,23 @@ class KOLLAInstallTask(Thread):
 
     def _run(self):
         # check and get version
-        cluster_data = registry.get_cluster_metadata(self.req.context, self.cluster_id)
+        cluster_data = registry.get_cluster_metadata(self.req.context,
+                                                     self.cluster_id)
         if cluster_data.get('tecs_version_id', None):
             vid = cluster_data['tecs_version_id']
             version_info = registry.get_version_metadata(self.req.context, vid)
             kolla_version_pkg_file = \
-                kolla_cmn.check_and_get_kolla_version(daisy_kolla_ver_path, version_info['name'])
+                kolla_cmn.check_and_get_kolla_version(daisy_kolla_ver_path,
+                                                      version_info['name'])
         else:
             kolla_version_pkg_file =\
-                kolla_cmn.check_and_get_tecs_version(daisy_kolla_ver_path)
+                kolla_cmn.check_and_get_kolla_version(daisy_kolla_ver_path)
         if not kolla_version_pkg_file:
             self.state = kolla_state['INSTALL_FAILED']
             self.message =\
                 "kolla version file not found in %s" % daisy_kolla_ver_path
             raise exception.NotFound(message=self.message)
+        kolla_cmn.version_load(kolla_version_pkg_file)
         (kolla_config, self.mgt_ip_list, host_name_ip_list) = \
             get_cluster_kolla_config(self.req, self.cluster_id)
         if not self.mgt_ip_list:
@@ -496,7 +499,6 @@ class KOLLAInstallTask(Thread):
                                        self.message, 0)
         docker_registry_ip = _get_local_ip()
         with open(self.log_file, "w+") as fp:
-            kolla_cmn.version_load(kolla_version_pkg_file, fp)
             threads = []
             for host in hosts_list:
                 t = threading.Thread(target=thread_bin,
@@ -607,5 +609,6 @@ class KOLLAInstallTask(Thread):
                                       kolla_state['ACTIVE'], 100)
                 for host_id in host_id_list:
                     daisy_cmn.update_db_host_status(
-                        self.req, host_id, {'tecs_version_id': cluster_data['tecs_version_id'],
+                        self.req, host_id,
-                                            'tecs_patch_id': ''})
+                        {'tecs_version_id': cluster_data['tecs_version_id'],
+                         'tecs_patch_id': ''})

code/daisy/daisy/api/backends/kolla/upgrade.py

@@ -18,7 +18,6 @@
 """
 
 import subprocess
-import time
 from oslo_log import log as logging
 from daisy import i18n
 import daisy.api.backends.common as daisy_cmn
@@ -62,11 +61,11 @@ class KOLLAUpgradeTask(Thread):
         self.kolla_file = "/home/kolla_install"
         self.log_file = "/var/log/daisy/kolla_%s_upgrade.log" % self.cluster_id
 
-
     def run(self):
         hosts = registry.get_cluster_hosts(self.req.context, self.cluster_id)
         hosts_id_list = [host['host_id'] for host in hosts]
-        cluster_meta = registry.get_cluster_metadata(self.req.context, self.cluster_id)
+        cluster_meta = registry.get_cluster_metadata(self.req.context,
+                                                     self.cluster_id)
         self.message = "prechecking envirnoment"
         update_all_host_progress_to_db(self.req, hosts_id_list,
                                        {'progress': 0,
@@ -79,7 +78,8 @@ class KOLLAUpgradeTask(Thread):
                 % kolla_cmn.daisy_kolla_path
             update_all_host_progress_to_db(self.req, hosts_id_list,
                                            {'progress': 0,
-                                            'status': kolla_state['UPDATE_FAILED'],
+                                            'status': kolla_state[
+                                                'UPDATE_FAILED'],
                                             'messages': self.message})
             raise exception.NotFound(message=self.message)
         if cluster_meta['tecs_version_id']:
@@ -88,12 +88,20 @@ class KOLLAUpgradeTask(Thread):
             if version_data['name'] == self.update_file:
                 LOG.error(_("kolla version %s is not need to upgrade!"
                             % version_data['name']))
-                self.message = "kolla version %s is not need to upgrade!" % version_data['name']
+                self.message = "kolla version %s is not need to upgrade!" \
+                    % version_data['name']
                 update_all_host_progress_to_db(self.req, hosts_id_list,
                                                {'progress': 0,
-                                                'status': kolla_state['UPDATE_FAILED'],
+                                                'status': kolla_state[
+                                                    'UPDATE_FAILED'],
                                                 'messages': self.message})
                 return
+        kolla_cmn.version_load(kolla_version_pkg_file)
+        update_all_host_progress_to_db(self.req, hosts_id_list,
+                                       {'progress': 10,
+                                        'status': kolla_state[
+                                            'UPDATING'],
+                                        'messages': self.message})
         for host in hosts:
             host_meta = daisy_cmn.get_host_detail(self.req, host["host_id"])
             host_ip = daisy_cmn.get_management_ip(host_meta)
@@ -104,23 +112,20 @@ class KOLLAUpgradeTask(Thread):
             if unreached_hosts:
                 self.message = "hosts %s ping failed" % unreached_hosts
                 update_all_host_progress_to_db(self.req, hosts_id_list,
-                                               {'progress': 0,
+                                               {'progress': 10,
-                                                'status': kolla_state['UPDATE_FAILED'],
+                                                'status': kolla_state[
+                                                    'UPDATE_FAILED'],
                                                 'messages': self.message})
                 raise exception.NotFound(message=self.message)
 
         LOG.info(_("precheck envirnoment successfully ..."))
         self.message = "openstack upgrading"
         update_all_host_progress_to_db(self.req, hosts_id_list,
-                                       {'progress': 10,
+                                       {'progress': 20,
-                                        'status': kolla_state['UPDATING'],
+                                        'status': kolla_state[
+                                            'UPDATING'],
                                         'messages': self.message})
         with open(self.log_file, "w+") as fp:
-            kolla_cmn.version_load(kolla_version_pkg_file, fp)
-            update_all_host_progress_to_db(self.req, hosts_id_list,
-                                           {'progress': 20,
-                                            'status': kolla_state['UPDATING'],
-                                            'messages': self.message})
             try:
                 LOG.info(_("begin to kolla-ansible "
                            "upgrade for all nodes..."))
@@ -134,7 +139,8 @@ class KOLLAUpgradeTask(Thread):
                 self.message = "kolla-ansible upgrade failed!"
                 update_all_host_progress_to_db(self.req, hosts_id_list,
                                                {'progress': 20,
-                                                'status': kolla_state['UPDATE_FAILED'],
+                                                'status': kolla_state[
+                                                    'UPDATE_FAILED'],
                                                 'messages': self.message})
                 LOG.info(_("kolla-ansible upgrade failed!"))
                 fp.write(e.output.strip())
@@ -145,7 +151,8 @@ class KOLLAUpgradeTask(Thread):
                 self.message = "openstack upgraded successfully"
                 update_all_host_progress_to_db(self.req, hosts_id_list,
                                                {'progress': 100,
-                                                'status': kolla_state['ACTIVE'],
+                                                'status': kolla_state[
+                                                    'ACTIVE'],
                                                 'messages': self.message})
                 for host_id in hosts_id_list:
                     daisy_cmn.update_db_host_status(
@@ -157,4 +164,3 @@ class KOLLAUpgradeTask(Thread):
                     self.req.context, self.cluster_id, cluster_meta)
                 LOG.info(_("openstack upgraded for cluster %s successfully."
                            % self.cluster_id))
-

code/daisy/daisy/tests/api/backends/kolla/test_install.py

@@ -434,20 +434,21 @@ class TestInstall(test.TestCase):
             mock_do_get_roles_detail,
             mock_do_get_hosts_of_role, mock_do_get_host_detail,
             mock_do_get_controller_node_cfg, mock_do_get_computer_node_cfg):
-        cmd = 'mkdir -p /home/kolla_install/docker/'
+        daisy_kolla_ver_path = '/var/lib/daisy/versionfile/kolla/'
+        cmd = 'mkdir -p %s' % daisy_kolla_ver_path
         subprocesscall(cmd)
-        cmd1 = 'rm -rf /home/kolla_install/docker/test.version'
+        cmd1 = 'rm -rf %s/test.version' % daisy_kolla_ver_path
         subprocesscall(cmd1)
-        cmd2 = 'touch /home/kolla_install/docker/test.version'
+        cmd2 = 'touch %s/test.version' % daisy_kolla_ver_path
         subprocesscall(cmd2)
-        f1 = open('/home/kolla_install/docker/test.version', 'a')
+        f1 = open('%s/test.version' % daisy_kolla_ver_path, 'a')
         f1.write('tag = 3.0.2')
         f1.close()
-        cmd3 = 'rm -rf /home/kolla_install/docker/all.yml'
+        cmd3 = 'rm -rf %s/all.yml' % daisy_kolla_ver_path
         subprocesscall(cmd3)
-        cmd4 = 'touch /home/kolla_install/docker/all.yml'
+        cmd4 = 'touch %s/all.yml' % daisy_kolla_ver_path
         subprocesscall(cmd4)
-        f2 = open('/home/kolla_install/docker/all.yml', 'a')
+        f2 = open('%s/all.yml' % daisy_kolla_ver_path, 'a')
         f2.write('openstack_release : 3.0.2')
         f2.close()
         mock_do__get_local_ip.return_value = '127.0.0.1'
@@ -468,12 +469,17 @@ class TestInstall(test.TestCase):
             install.get_cluster_kolla_config(
             self.req,
             '8ad27e36-f3e2-48b4-84b8-5b676c6fabde')
-        cmd_end1 = 'rm -rf /home/kolla_install/docker/test.version'
+        cmd_end1 = 'rm -rf %s/test.version' % daisy_kolla_ver_path
         subprocesscall(cmd_end1)
-        cmd_end2 = 'rm -rf /home/kolla_install/docker/all.yml'
+        cmd_end2 = 'rm -rf %s/all.yml' % daisy_kolla_ver_path
         subprocesscall(cmd_end2)
         self.assertEqual('3.0.2', kolla_config['Version'])
 
+    @mock.patch('daisy.api.backends.common.update_db_host_status')
+    @mock.patch("daisy.registry.client.v1.api.get_version_metadata")
+    @mock.patch('daisy.api.backends.kolla.common.version_load')
+    @mock.patch('daisy.api.backends.kolla.common.check_and_get_kolla_version')
+    @mock.patch("daisy.registry.client.v1.api.get_cluster_metadata")
     @mock.patch('daisy.api.backends.kolla.install._get_local_ip')
     @mock.patch('daisy.api.common.config_network_new')
     @mock.patch('daisy.registry.client.v1.client.RegistryClient.do_request')
@@ -501,7 +507,10 @@ class TestInstall(test.TestCase):
             mock_do_get_host_detail, mock_do_get_assigned_network,
             mock_do_update_all_host_progress_to_db, mock_do_config_nodes_hosts,
             mock_do_update_host_progress_to_db, mock_do_update_progress_to_db,
-            mock_do_request, mock_do_config_network_new, mock_do_get_local_ip):
+            mock_do_request, mock_do_config_network_new,
+            mock_do_get_local_ip, mock_do_get_clusters_detail,
+            mock_do_check_and_get_kolla_version, mock_do_version_load,
+            mock_do_get_version_metadata, mock_do_update_db_host_status):
 
         def mock_get_cluster_kolla_config(*args, **kwargs):
             return (kolla_config, mgt_ip_list, host_name_ip_list)
@@ -517,6 +526,13 @@ class TestInstall(test.TestCase):
                 get_result = {'host': host6_meta}
                 res.read.return_value = jsonutils.dumps(get_result)
                 return res
+
+        def mock_get_clusters_detail(*args, **kwargs):
+            return {
+                "id": "93ca3165-1a82-4c4a-914f-65279827e46e",
+                "name": "test",
+                "tecs_version_id": "1111111"}
+
         cmd = 'mkdir -p /var/log/daisy'
         subprocesscall(cmd)
         mock_do_request.side_effect = fake_do_request
@@ -528,6 +544,10 @@ class TestInstall(test.TestCase):
         mock_do_get_host_detail.return_value = host6_meta
         mock_do_get_assigned_network.return_value = assigned_network
         mock_do_get_local_ip.return_value = '127.0.0.1'
+        mock_do_get_clusters_detail.side_effect = mock_get_clusters_detail
+        mock_do_check_and_get_kolla_version.return_value = \
+            "/var/lib/daisy/versionfile/kolla/test_version"
+        mock_do_get_version_metadata.return_value = {"name": "test_version"}
         kolla_config = {}
         mgt_ip_list = ['127.0.0.1']
         host_name_ip_list = [{'localhost': '127.0.0.1'}]

code/daisyclient/daisyclient/v1/shell.py

@@ -345,7 +345,7 @@ def do_host_list(gc, args):
 
     columns = ['ID', 'Name', 'Description', 'Resource_type',
                'Status', 'Os_progress', 'Os_status', 'Discover_state',
-               'Messages']
+               'Tecs_version_id', 'Messages']
 #    if filters.has_key('cluster_id'):
     if 'cluster_id' in filters:
         role_columns = ['Role_progress', 'Role_status', 'Role_messages']

code/horizon/openstack_dashboard/dashboards/environment/cluster/net_plane.py

@@ -83,7 +83,7 @@ def set_net_plane(request, cluster_id, nets):
 @csrf_exempt
 def sort_net_planes(net_planes):
     ret_net_planes = []
-    sort_list = ["MANAGEMENT", "PUBLICAPI", "DATAPLANE","EXTERNAL",
+    sort_list = ["MANAGEMENT", "PUBLICAPI", "DATAPLANE", "EXTERNAL",
                  "STORAGE", "VXLAN", "HEARTBEAT"]
     for sort in sort_list:
         for net_plane in net_planes:
@@ -105,7 +105,7 @@ def get_default_net_plane():
         {"network_type": "STORAGE",
          "net_planes": [{"name": "STORAGE", }]},
         {"network_type": "EXTERNAL",
-         "net_planes": [{"name": "EXTERNAL", }]},]
+         "net_planes": [{"name": "EXTERNAL", }]}, ]
     for net in networks:
         for net_plane in net["net_planes"]:
             net_plane.update({"cidr": "192.168.1.1/24",
@@ -196,7 +196,7 @@ def delete_net_plane(request, cluster_id):
 @csrf_exempt
 def get_net_plane_list(request, cluster_id):
     ret_net_planes = []
-    filter_net_planes = ["DEPLOYMENT",]
+    filter_net_planes = ["DEPLOYMENT", ]
     try:
         network_list = api.daisy.network_list(request, cluster_id)
         show_net_planes = [net.__dict__ for net in network_list

code/horizon/openstack_dashboard/dashboards/environment/cluster/tables.py

@@ -187,7 +187,6 @@ class HostsTable(tables.DataTable):
                                 verbose_name=_('host_status'),
                                 hidden=True)
 
-
     def __init__(self, request, data=None, needs_form_wrapper=None, **kwargs):
         super(HostsTable, self).__init__(request,
                                          data,

code/horizon/openstack_dashboard/dashboards/environment/cluster/views.py

@@ -72,8 +72,8 @@ def count_deploy_info(request, host_list):
         if len(i18n_list) > 1:
             status_info = "%s, %s" % (i18n_list[0], i18n_list[1])
 
-        #message = deploy_info.get("role_message", "")
+        # message = deploy_info.get("role_message", "")
-        #analyze_result = cluster_tables.analyze_role_msg(message)
+        # analyze_result = cluster_tables.analyze_role_msg(message)
 
         host = {
             "id": host.id,
@@ -170,7 +170,7 @@ class ClusterView(tables.DataTableView):
             if not hasattr(host, 'role_status'):
                 host.role_status = ""
             backends = host_views.get_backend_type_by_role_list(self.request)
-           
+
             host.current_version = []
             if hasattr(host, 'os_version_id') and host.os_version_id:
                 version = api.daisy.version_get(self.request,
@@ -288,6 +288,7 @@ def check_all_tecs_install_status(request, cluster_id):
 
     return error_host_names
 
+
 def uninstall_version(request, cluster_id):
     try:
         api.daisy.uninstall_cluster(request, cluster_id)

code/horizon/openstack_dashboard/dashboards/environment/deploy/network_mapping_views.py

@@ -183,7 +183,7 @@ class NetworkMappingView(tables.DataTableView):
     def get_net_plane(self):
         network_data = []
         try:
-            filter_net_planes = ["DEPLOYMENT",]
+            filter_net_planes = ["DEPLOYMENT", ]
             network_list = api.daisy.network_list(self.request,
                                                   self.kwargs["cluster_id"])
             for network in network_list:

code/horizon/openstack_dashboard/dashboards/environment/version/views.py

@@ -511,7 +511,7 @@ class UpdateVersionForm(forms.SelfHandlingForm):
 
     FILE_TYPE_CHOICES = [
         ("Centos7", "Centos7"),
-        ("kolla", "kolla"),]
+        ("kolla", "kolla"), ]
     VERSION_TYPE_CHOICES = [
         ("system", _("System version")),
         ("patch", _("Patch version"))]

tools/setup/install/install_func.sh

@@ -97,17 +97,23 @@ function kolla_install
   imageversion="latest"
   imageserver="http://120.24.17.215"
   imagedir="/var/lib/daisy/versionfile/kolla"
+  imagebakdir="/home/kolla_install/docker/"
   imagename="kolla-image-$imagebranch-$imageversion.tgz"
   sourcedir="/home/kolla_install/"
 
   write_install_log "Begin copy images..."
-  cd $imagedir
+  if [ ! -f "$imagedir/$imagename" ];then
-  if [ -f "$imagedir/$imagename" ];then
+      mkdir -p $imagebakdir
-      echo "$imagename already exist!"
+      cd $imagebakdir
-  else
+      if [ -f "$imagebakdir/$imagename" ];then
-      wget "$imageserver/$imagename"
+          echo "$imagename already exist!"
+      else
+          wget "$imageserver/$imagename"
+      fi
+      cp $imagebakdir/$imagename $imagedir
   fi
-  tar mzxvf $imagename
+  cd $imagedir
+  tar mzxvf $imagedir/$imagename
   sourceversion=$(cat $imagedir/registry-*.version | head -1)
 
   write_install_log "Begin clone kolla... $sourceversion"
@@ -124,11 +130,14 @@ function kolla_install
   cp -r /home/kolla_install/kolla/etc/kolla /etc
 
   # TODO: (huzhj)Use latest registry server from upstream
-  cd $imagedir
+  if [ ! -f "$imagedir/registry-server.tar" ];then
-  if [ -f "$imagedir/registry-server.tar" ];then
+      cd $imagebakdir
-      echo "registry-server.tar already exist!"
+      if [ -f "$imagebakdir/registry-server.tar" ];then
-  else
+          echo "registry-server.tar already exist!"
-      wget "http://daisycloud.org/static/files/registry-server.tar"
+      else
+          wget "http://daisycloud.org/static/files/registry-server.tar"
+      fi
+      cp $imagebakdir/registry-server.tar $imagedir
   fi
   docker load < ./registry-server.tar
   rm -rf $imagedir/tmp