Merge remote-tracking branch 'starlingx/master' into HEAD

Change-Id: I0b2c69e122d783cf98f37c29d2f4a56e8e337b00
Signed-off-by: Scott Little <scott.little@windriver.com>

nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py

@@ -134,9 +134,11 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
 
         openstack_compute = False
         openstack_control = False
+        remote_storage = False
 
         OS_COMPUTE = nfvi.objects.v1.HOST_LABEL_KEYS.OS_COMPUTE_NODE
         OS_CONTROL = nfvi.objects.v1.HOST_LABEL_KEYS.OS_CONTROL_PLANE
+        REMOTE_STORAGE = nfvi.objects.v1.HOST_LABEL_KEYS.REMOTE_STORAGE
         LABEL_ENABLED = nfvi.objects.v1.HOST_LABEL_VALUES.ENABLED
 
         for host_label in host_label_list:
@@ -147,8 +149,11 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
             elif host_label['label_key'] == OS_CONTROL:
                 if host_label['label_value'] == LABEL_ENABLED:
                     openstack_control = True
+            elif host_label['label_key'] == REMOTE_STORAGE:
+                if host_label['label_value'] == LABEL_ENABLED:
+                    remote_storage = True
 
-        return (openstack_compute, openstack_control)
+        return (openstack_compute, openstack_control, remote_storage)
 
     def __init__(self):
         super(NFVIInfrastructureAPI, self).__init__()
@@ -378,7 +383,8 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
 
                 host_label_list = future.result.data['labels']
 
-                openstack_compute, openstack_control = self._get_host_labels(host_label_list)
+                openstack_compute, openstack_control, remote_storage = \
+                    self._get_host_labels(host_label_list)
 
                 admin_state, oper_state, avail_status, nfvi_data \
                     = host_state(host_uuid, host_name, host_personality,
@@ -400,6 +406,7 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
                                                 target_load,
                                                 openstack_compute,
                                                 openstack_control,
+                                                remote_storage,
                                                 nfvi_data)
 
                 host_objs.append(host_obj)
@@ -520,7 +527,8 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
 
             host_label_list = future.result.data['labels']
 
-            openstack_compute, openstack_control = self._get_host_labels(host_label_list)
+            openstack_compute, openstack_control, remote_storage = \
+                self._get_host_labels(host_label_list)
 
             host_obj = nfvi.objects.v1.Host(host_uuid, host_name,
                                             host_sub_functions,
@@ -532,6 +540,7 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
                                             target_load,
                                             openstack_compute,
                                             openstack_control,
+                                            remote_storage,
                                             nfvi_data)
 
             response['result-data'] = host_obj
@@ -884,34 +893,21 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
             future.set_timeouts(config.CONF.get('nfvi-timeouts', None))
 
             if self._host_supports_kubernetes(host_personality):
-                if True:
+                response['reason'] = 'failed to disable kubernetes services'
-                    # For now, we do not want to apply the NoExecute taint.
-                    # When the VIM detects that a service is failed on a host,
-                    # it goes through a disable/enable cycle. This would cause
-                    # the NoExecute taint to be applied/removed which causes
-                    # most pods to be stopped/started. If the pods don't come
-                    # back quickly enough the VIM will attempt another
-                    # disable/enable, which can go on forever. For now,
-                    # we will just avoid tainting hosts.
-                    # TODO(bwensley): Rework when support for pure k8s hosts is
-                    # added.
-                    pass
-                else:
-                    response['reason'] = 'failed to disable kubernetes services'
 
-                    # To disable kubernetes we add the NoExecute taint to the
+                # To disable kubernetes we add the NoExecute taint to the
-                    # node. This removes pods that can be scheduled elsewhere
+                # node. This removes pods that can be scheduled elsewhere
-                    # and prevents new pods from scheduling on the node.
+                # and prevents new pods from scheduling on the node.
-                    future.work(kubernetes_client.taint_node,
+                future.work(kubernetes_client.taint_node,
-                                host_name, "NoExecute", "services", "disabled")
+                            host_name, "NoExecute", "services", "disabled")
 
-                    future.result = (yield)
+                future.result = (yield)
 
-                    if not future.result.is_complete():
+                if not future.result.is_complete():
-                        DLOG.error("Kubernetes taint_node failed, operation "
+                    DLOG.error("Kubernetes taint_node failed, operation "
-                                   "did not complete, host_uuid=%s, host_name=%s."
+                               "did not complete, host_uuid=%s, host_name=%s."
-                                   % (host_uuid, host_name))
+                               % (host_uuid, host_name))
-                        return
+                    return
 
             response['completed'] = True
             response['reason'] = ''
@@ -1006,7 +1002,8 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
 
             host_label_list = future.result.data['labels']
 
-            openstack_compute, openstack_control = self._get_host_labels(host_label_list)
+            openstack_compute, openstack_control, remote_storage = \
+                self._get_host_labels(host_label_list)
 
             host_obj = nfvi.objects.v1.Host(host_uuid, host_name,
                                             host_sub_functions,
@@ -1018,6 +1015,7 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
                                             target_load,
                                             openstack_compute,
                                             openstack_control,
+                                            remote_storage,
                                             nfvi_data)
 
             response['result-data'] = host_obj
@@ -1122,7 +1120,8 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
 
             host_label_list = future.result.data['labels']
 
-            openstack_compute, openstack_control = self._get_host_labels(host_label_list)
+            openstack_compute, openstack_control, remote_storage = \
+                self._get_host_labels(host_label_list)
 
             host_obj = nfvi.objects.v1.Host(host_uuid, host_name,
                                             host_sub_functions,
@@ -1134,6 +1133,7 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
                                             target_load,
                                             openstack_compute,
                                             openstack_control,
+                                            remote_storage,
                                             nfvi_data)
 
             response['result-data'] = host_obj
@@ -1237,7 +1237,8 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
 
             host_label_list = future.result.data['labels']
 
-            openstack_compute, openstack_control = self._get_host_labels(host_label_list)
+            openstack_compute, openstack_control, remote_storage = \
+                self._get_host_labels(host_label_list)
 
             host_obj = nfvi.objects.v1.Host(host_uuid, host_name,
                                             host_sub_functions,
@@ -1249,6 +1250,7 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
                                             target_load,
                                             openstack_compute,
                                             openstack_control,
+                                            remote_storage,
                                             nfvi_data)
 
             response['result-data'] = host_obj
@@ -1351,7 +1353,8 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
 
             host_label_list = future.result.data['labels']
 
-            openstack_compute, openstack_control = self._get_host_labels(host_label_list)
+            openstack_compute, openstack_control, remote_storage = \
+                self._get_host_labels(host_label_list)
 
             host_obj = nfvi.objects.v1.Host(host_uuid, host_name,
                                             host_sub_functions,
@@ -1363,6 +1366,7 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
                                             target_load,
                                             openstack_compute,
                                             openstack_control,
+                                            remote_storage,
                                             nfvi_data)
 
             response['result-data'] = host_obj
@@ -1518,7 +1522,8 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
 
             host_label_list = future.result.data['labels']
 
-            openstack_compute, openstack_control = self._get_host_labels(host_label_list)
+            openstack_compute, openstack_control, remote_storage = \
+                self._get_host_labels(host_label_list)
 
             host_obj = nfvi.objects.v1.Host(host_uuid, host_name,
                                             host_sub_functions,
@@ -1530,6 +1535,7 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
                                             target_load,
                                             openstack_compute,
                                             openstack_control,
+                                            remote_storage,
                                             nfvi_data)
 
             response['result-data'] = host_obj

nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/neutron.py

@@ -669,49 +669,45 @@ def query_network_agents(token, host_name, check_fully_up):
     Input parameter check_fully_up set to True will check for
     both alive and admin_state_up, otherwise only alive is checked.
     """
-    try:
+    url, api_cmd, api_cmd_headers, result_data = get_network_agents(
-        url, api_cmd, api_cmd_headers, result_data = get_network_agents(
+        token, host_name)
-            token, host_name)
 
-        agent_state = 'up'
+    agent_state = 'up'
-        supported_agents = [AGENT_TYPE.L3, AGENT_TYPE.DHCP]
+    alive = False
-        for supported_agent in supported_agents:
+    admin_state_up = False
-            found = False
+    supported_agents = [AGENT_TYPE.L3, AGENT_TYPE.DHCP]
-            for agent in result_data:
+    for supported_agent in supported_agents:
-                agent_type = agent.get('agent_type', '')
+        found = False
-                host = agent.get('host', '')
+        for agent in result_data:
-                if (agent_type == supported_agent) and (host == host_name):
+            agent_type = agent.get('agent_type', '')
-                    DLOG.verbose("found agent %s for host %s" %
+            host = agent.get('host', '')
-                                 (supported_agent, host_name))
+            if (agent_type == supported_agent) and (host == host_name):
-                    alive = agent.get('alive', False)
+                DLOG.verbose("found agent %s for host %s" %
-                    admin_state_up = agent.get('admin_state_up', False)
+                             (supported_agent, host_name))
-                    # found the agent of interest.
+                alive = agent.get('alive', False)
-                    found = True
+                admin_state_up = agent.get('admin_state_up', False)
-                    break
+                # found the agent of interest.
-            if found:
+                found = True
-                if check_fully_up:
-                    if not (alive and admin_state_up):
-                        DLOG.verbose("host %s agent %s not fully up. alive: %s,"
-                                     " admin_state_up: %s" %
-                                     (host_name, supported_agent,
-                                      alive, admin_state_up))
-                        agent_state = 'down'
-                        break
-                else:
-                    if not alive:
-                        DLOG.verbose("host %s agent %s not alive" %
-                                     (host_name, supported_agent))
-                        agent_state = 'down'
-                        break
-            else:
-                DLOG.error("host %s agent %s not present" %
-                           (host_name, supported_agent))
-                agent_state = 'down'
                 break
-
+        if found:
-    except Exception as e:
+            if check_fully_up:
-        DLOG.exception("Caught exception trying to query host %s "
+                if not (alive and admin_state_up):
-                       "agent states: %s" % (host_name, e))
+                    DLOG.verbose("host %s agent %s not fully up. alive: %s,"
-        agent_state = 'down'
+                                 " admin_state_up: %s" %
+                                 (host_name, supported_agent,
+                                  alive, admin_state_up))
+                    agent_state = 'down'
+                    break
+            else:
+                if not alive:
+                    DLOG.verbose("host %s agent %s not alive" %
+                                 (host_name, supported_agent))
+                    agent_state = 'down'
+                    break
+        else:
+            DLOG.error("host %s agent %s not present" %
+                       (host_name, supported_agent))
+            agent_state = 'down'
+            break
 
     return agent_state

nfv/nfv-tests/nfv_unit_tests/tests/test_instance.py

@@ -217,6 +217,7 @@ class TestInstance(testcase.NFVTestCase):
             target_load=target_load,
             openstack_compute=False,
             openstack_control=False,
+            remote_storage=False,
             uptime='1000'
         )
 

nfv/nfv-tests/nfv_unit_tests/tests/test_sw_patch_strategy.py

@@ -296,6 +296,7 @@ class TestSwPatchStrategy(testcase.NFVTestCase):
             target_load='12.01',
             openstack_compute=False,
             openstack_control=False,
+            remote_storage=False,
             uptime='1000'
         )
 

nfv/nfv-tests/nfv_unit_tests/tests/test_sw_upgrade_strategy.py

@@ -274,6 +274,7 @@ class TestSwUpgradeStrategy(testcase.NFVTestCase):
             target_load=target_load,
             openstack_compute=False,
             openstack_control=False,
+            remote_storage=False,
             uptime='1000'
         )
 

nfv/nfv-vim/nfv_vim/database/_database_infrastructure_module.py

@@ -126,6 +126,7 @@ def database_host_get_list():
                                          nfvi_host_data['target_load'],
                                          nfvi_host_data['openstack_compute'],
                                          nfvi_host_data['openstack_control'],
+                                         nfvi_host_data['remote_storage'],
                                          nfvi_host_data['nfvi_data'])
 
         host_obj = objects.Host(nfvi_host, host.state, host.action,

nfv/nfv-vim/nfv_vim/database/_database_migrate.py

@@ -23,6 +23,7 @@ def _migrate_hosts_v5_to_v6(session, hosts_v5, hosts_v6):
             nfvi_host_data = json.loads(host_v5.nfvi_host_data)
             nfvi_host_data['openstack_compute'] = False
             nfvi_host_data['openstack_control'] = False
+            nfvi_host_data['remote_storage'] = False
             host_v6.nfvi_host_data = json.dumps(nfvi_host_data)
             session.add(host_v6)
 

nfv/nfv-vim/nfv_vim/directors/_sw_mgmt_director.py

@@ -38,6 +38,13 @@ class SwMgmtDirector(object):
         """
         return self._sw_update
 
+    @property
+    def single_controller(self):
+        """
+        Returns whether this is a single controller configuration
+        """
+        return self._single_controller
+
     def create_sw_patch_strategy(self, controller_apply_type, storage_apply_type,
                                  swift_apply_type, worker_apply_type,
                                  max_parallel_worker_hosts,

nfv/nfv-vim/nfv_vim/host_fsm/_host_tasks.py

@@ -234,9 +234,6 @@ class DisableHostTask(state_machine.StateTask):
         if host.host_service_configured(objects.HOST_SERVICES.GUEST):
             task_work_list.append(DisableHostServicesTaskWork(
                 self, host, objects.HOST_SERVICES.GUEST))
-        if host.host_service_configured(objects.HOST_SERVICES.CONTAINER):
-            task_work_list.append(DisableHostServicesTaskWork(
-                self, host, objects.HOST_SERVICES.CONTAINER))
         if host.host_service_configured(objects.HOST_SERVICES.COMPUTE):
             task_work_list.append(QueryHypervisorTaskWork(
                 self, host, force_pass=True))
@@ -248,6 +245,17 @@ class DisableHostTask(state_machine.StateTask):
             task_work_list.append(NotifyHostDisabledTaskWork(
                 self, host, objects.HOST_SERVICES.NETWORK))
         task_work_list.append(NotifyInstancesHostDisabledTaskWork(self, host))
+        if host.host_service_configured(objects.HOST_SERVICES.CONTAINER):
+            # Only disable the container services if the host is being locked
+            # and we are not running in a single controller configuration. In
+            # a single controller configuration we keep the container services
+            # running.
+            if self._host.is_locking():
+                from nfv_vim import directors
+                sw_mgmt_director = directors.get_sw_mgmt_director()
+                if not sw_mgmt_director.single_controller:
+                    task_work_list.append(DisableHostServicesTaskWork(
+                        self, host, objects.HOST_SERVICES.CONTAINER))
         task_work_list.append(notify_host_services_task(
             self, host, force_pass=True))
         if host.host_service_configured(objects.HOST_SERVICES.COMPUTE):
@@ -443,8 +451,21 @@ class NotifyDisabledHostTask(state_machine.StateTask):
     Notify Disabled Host Task
     """
     def __init__(self, host):
+        from nfv_vim import objects
+
         self._host_reference = weakref.ref(host)
         task_work_list = list()
+        if host.host_service_configured(objects.HOST_SERVICES.CONTAINER):
+            # Only disable the container services if the host is being locked
+            # and we are not running in a single controller configuration. In
+            # a single controller configuration we keep the container services
+            # running.
+            if self._host.is_locking():
+                from nfv_vim import directors
+                sw_mgmt_director = directors.get_sw_mgmt_director()
+                if not sw_mgmt_director.single_controller:
+                    task_work_list.append(DisableHostServicesTaskWork(
+                        self, host, objects.HOST_SERVICES.CONTAINER))
         task_work_list.append(NotifyHostServicesDisabledTaskWork(
             self, host, force_pass=True))
         super(NotifyDisabledHostTask, self).__init__(

nfv/nfv-vim/nfv_vim/nfvi/objects/v1/_host.py

@@ -88,6 +88,7 @@ class KubernetesLabelKeys(Constants):
     """
     OS_COMPUTE_NODE = Constant('openstack-compute-node')
     OS_CONTROL_PLANE = Constant('openstack-control-plane')
+    REMOTE_STORAGE = Constant('remote-storage')
 
 
 # Host Constant Instantiation
@@ -108,6 +109,7 @@ class Host(ObjectData):
                  avail_status, action, uptime, software_load, target_load,
                  openstack_compute=False,
                  openstack_control=False,
+                 remote_storage=False,
                  nfvi_data=None):
         super(Host, self).__init__('1.0.0')
         self.update(dict(uuid=uuid, name=name, personality=personality,
@@ -119,6 +121,7 @@ class Host(ObjectData):
                          software_load=software_load,
                          target_load=target_load,
                          openstack_compute=openstack_compute,
-                         openstack_control=openstack_control))
+                         openstack_control=openstack_control,
+                         remote_storage=remote_storage))
 
         self.nfvi_data = nfvi_data

nfv/nfv-vim/nfv_vim/objects/_host.py

@@ -317,6 +317,13 @@ class Host(ObjectData):
         """
         return self._nfvi_host.openstack_control
 
+    @property
+    def remote_storage(self):
+        """
+        Returns remote_storage for this host
+        """
+        return self._nfvi_host.remote_storage
+
     @property
     def recover_instances(self):
         """

nfv/nfv-vim/nfv_vim/objects/_instance.py

@@ -1456,6 +1456,8 @@ class Instance(ObjectData):
         """
         Returns true if the instance can be cold-migrated
         """
+        from nfv_vim import tables
+
         if not system_initiated:
             # Always allow user initiated cold migration
             return True
@@ -1464,9 +1466,14 @@ class Instance(ObjectData):
             # Always allow cold migration when booted from a volume
             return True
 
-        # TODO(bwensley): Always allow cold migration for instances using
+        host_table = tables.tables_get_host_table()
-        # remote storage. There is currently no way to determine this, but we
+        host = host_table.get(self.host_name, None)
-        # should eventually be able to check for a label on the compute host.
+
+        if host is not None:
+            if host.remote_storage:
+                # Always allow cold migration for instances using
+                # remote storage
+                return True
 
         config_option = 'max_cold_migrate_local_image_disk_gb'
 
@@ -1487,6 +1494,8 @@ class Instance(ObjectData):
         """
         Returns true if the instance can be evacuated
         """
+        from nfv_vim import tables
+
         if not system_initiated:
             # Always allow user initiated evacuate
             return True
@@ -1495,9 +1504,14 @@ class Instance(ObjectData):
             # Always allow evacuate when booted from a volume
             return True
 
-        # TODO(bwensley): Always allow evacuate for instances using remote
+        host_table = tables.tables_get_host_table()
-        # storage. There is currently no way to determine this, but we should
+        host = host_table.get(self.host_name, None)
-        # eventually be able to check for a label on the compute host.
+
+        if host is not None:
+            if host.remote_storage:
+                # Always allow evacuation for instances using
+                # remote storage
+                return True
 
         config_option = 'max_evacuate_local_image_disk_gb'