diff --git a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_network_api.py b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_network_api.py index b1a0a70c..97a04b3f 100755 --- a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_network_api.py +++ b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_network_api.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: Apache-2.0 # +import os from six.moves import http_client as httplib from nfv_common import debug @@ -91,6 +92,15 @@ class NFVINetworkAPI(nfvi.api.v1.NFVINetworkAPI): (self._directory.get_service_info( OPENSTACK_SERVICE.NEUTRON) is not None)) + @staticmethod + def _host_supports_kubernetes(): + # TODO(ksmith): This check will disappear once kubernetes is the + # default + # Note we do not check personality as the assumption is that + # in the case of kubernetes, we won't even get to this check + # unless the correct label is applied to the host. + return (os.path.isfile('/etc/kubernetes/admin.conf')) + def get_networks(self, future, paging, callback): """ Get a list of networks @@ -775,6 +785,7 @@ class NFVINetworkAPI(nfvi.api.v1.NFVINetworkAPI): host_personality, callback): """ Create Host Services, notify neutron to create services for a host. + Only used in non k8s config. """ response = dict() response['completed'] = False @@ -940,14 +951,23 @@ class NFVINetworkAPI(nfvi.api.v1.NFVINetworkAPI): self._neutron_extensions = future.result.data - if neutron.lookup_extension(neutron.EXTENSION_NAMES.HOST, + if self._host_supports_kubernetes(): + extension_name = neutron.EXTENSION_NAMES.AGENT + else: + extension_name = neutron.EXTENSION_NAMES.HOST + + if neutron.lookup_extension(extension_name, self._neutron_extensions): response['reason'] = 'failed to delete neutron services' # Send the delete request to Neutron. - future.work(neutron.delete_host_services, - self._token, host_uuid) + if not self._host_supports_kubernetes(): + future.work(neutron.delete_host_services, + self._token, host_uuid) + else: + future.work(neutron.delete_network_agents, + self._token, host_name) try: future.result = (yield) @@ -1025,13 +1045,23 @@ class NFVINetworkAPI(nfvi.api.v1.NFVINetworkAPI): self._neutron_extensions = future.result.data - if neutron.lookup_extension(neutron.EXTENSION_NAMES.HOST, + if self._host_supports_kubernetes(): + extension_name = neutron.EXTENSION_NAMES.AGENT + else: + extension_name = neutron.EXTENSION_NAMES.HOST + + if neutron.lookup_extension(extension_name, self._neutron_extensions): response['reason'] = 'failed to enable neutron services' # Send the Enable request to Neutron - future.work(neutron.enable_host_services, - self._token, host_uuid) + if not self._host_supports_kubernetes(): + future.work(neutron.enable_host_services, + self._token, host_uuid) + else: + future.work(neutron.enable_network_agents, + self._token, host_name) + future.result = (yield) if not future.result.is_complete(): @@ -1041,14 +1071,21 @@ class NFVINetworkAPI(nfvi.api.v1.NFVINetworkAPI): % (host_uuid, host_name)) return - result_data = future.result.data['host'] - if not ('up' == result_data['availability'] and - host_name == result_data['name'] and - host_uuid == result_data['id']): - DLOG.error("Neutron enable-host-services failed, " - "operation did not complete, host_uuid=%s, " - "host_name=%s." % (host_uuid, host_name)) - return + if not self._host_supports_kubernetes(): + result_data = future.result.data['host'] + if not ('up' == result_data['availability'] and + host_name == result_data['name'] and + host_uuid == result_data['id']): + DLOG.error("Neutron enable-host-services failed, " + "operation did not complete, host_uuid=%s, " + "host_name=%s." % (host_uuid, host_name)) + return + else: + if not future.result.data: + DLOG.error("Neutron enable-host-services (agents) failed, " + "operation did not complete, host_uuid=%s, " + "host_name=%s." % (host_uuid, host_name)) + return response['completed'] = True response['reason'] = '' @@ -1073,7 +1110,8 @@ class NFVINetworkAPI(nfvi.api.v1.NFVINetworkAPI): callback.close() def query_host_services(self, future, host_uuid, host_name, - host_personality, callback): + host_personality, check_fully_up, + callback): """ Query Neutron Services for a host. """ @@ -1110,11 +1148,22 @@ class NFVINetworkAPI(nfvi.api.v1.NFVINetworkAPI): self._neutron_extensions = future.result.data - if neutron.lookup_extension(neutron.EXTENSION_NAMES.HOST, + if self._host_supports_kubernetes(): + extension_name = neutron.EXTENSION_NAMES.AGENT + else: + extension_name = neutron.EXTENSION_NAMES.HOST + + if neutron.lookup_extension(extension_name, self._neutron_extensions): # Send Query request to Neutron - future.work(neutron.query_host_services, - self._token, host_name) + if not self._host_supports_kubernetes(): + future.work(neutron.query_host_services, + self._token, host_name) + else: + future.work(neutron.query_network_agents, + self._token, host_name, + check_fully_up) + future.result = (yield) if not future.result.is_complete(): @@ -1189,14 +1238,24 @@ class NFVINetworkAPI(nfvi.api.v1.NFVINetworkAPI): self._neutron_extensions = future.result.data - if neutron.lookup_extension(neutron.EXTENSION_NAMES.HOST, + if self._host_supports_kubernetes(): + extension_name = neutron.EXTENSION_NAMES.AGENT + else: + extension_name = neutron.EXTENSION_NAMES.HOST + + if neutron.lookup_extension(extension_name, self._neutron_extensions): response['reason'] = 'failed to disable neutron services' # Send the Disable request to Neutron - future.work(neutron.disable_host_services, - self._token, - host_uuid) + if not self._host_supports_kubernetes(): + future.work(neutron.disable_host_services, + self._token, + host_uuid) + else: + future.work(neutron.disable_network_agents, + self._token, host_name) + future.result = (yield) if not future.result.is_complete(): @@ -1205,14 +1264,21 @@ class NFVINetworkAPI(nfvi.api.v1.NFVINetworkAPI): "host_name=%s." % (host_uuid, host_name)) return - result_data = future.result.data['host'] - if not ('down' == result_data['availability'] and - host_name == result_data['name'] and - host_uuid == result_data['id']): - DLOG.error("Neutron disable-host-services failed, " - "operation did not complete, host_uuid=%s, " - "host_name=%s." % (host_uuid, host_name)) - return + if not self._host_supports_kubernetes(): + result_data = future.result.data['host'] + if not ('down' == result_data['availability'] and + host_name == result_data['name'] and + host_uuid == result_data['id']): + DLOG.error("Neutron disable-host-services failed, " + "operation did not complete, host_uuid=%s, " + "host_name=%s." % (host_uuid, host_name)) + return + else: + if not future.result.data: + DLOG.error("Neutron disable-host-services (agents) failed, " + "operation did not complete, host_uuid=%s, " + "host_name=%s." % (host_uuid, host_name)) + return response['completed'] = True response['reason'] = '' diff --git a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/neutron.py b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/neutron.py index 1c4beb56..1d14fbd4 100755 --- a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/neutron.py +++ b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/neutron.py @@ -24,6 +24,7 @@ class NeutronExtensionNames(Constants): Neutron Extension Name Constants """ HOST = Constant('host') + AGENT = Constant('agent') @six.add_metaclass(Singleton) @@ -46,10 +47,38 @@ class NetworkStatus(Constants): ERROR = Constant('ERROR') +@six.add_metaclass(Singleton) +class AgentType(Constants): + """ + AGENT TYPE Constants + """ + L3 = Constant('L3 agent') + DHCP = Constant('DHCP agent') + + # Constant Instantiation EXTENSION_NAMES = NeutronExtensionNames() NETWORK_ADMIN_STATE = NetworkAdministrativeState() NETWORK_STATUS = NetworkStatus() +AGENT_TYPE = AgentType() + + +def get_network_agents(token, host_name): + + url = token.get_service_url(OPENSTACK_SERVICE.NEUTRON) + if url is None: + raise ValueError("OpenStack Neutron URL is invalid") + + api_cmd = url + "/v2.0/agents?host=" + host_name + + api_cmd_headers = dict() + api_cmd_headers['wrs-header'] = 'true' + api_cmd_headers['Content-Type'] = "application/json" + + response = rest_api_request(token, "GET", api_cmd, api_cmd_headers) + result_data = response.result_data['agents'] + + return url, api_cmd, api_cmd_headers, result_data def lookup_extension(extension_name, extensions): @@ -385,6 +414,36 @@ def delete_host_services(token, host_uuid): return response +def delete_network_agents(token, host_name): + """ + Asks OpenStack Neutron to delete agents for a host + """ + try: + url, api_cmd, api_cmd_headers, result_data = get_network_agents( + token, host_name) + + num_agents_found = 0 + supported_agents = [AGENT_TYPE.L3, AGENT_TYPE.DHCP] + for agent in result_data: + if (agent['host'] == host_name and + agent['agent_type'] in supported_agents): + api_cmd = url + "/v2.0/agents/%s" % agent['id'] + response = rest_api_request(token, "DELETE", api_cmd, + api_cmd_headers) + num_agents_found = num_agents_found + 1 + + except Exception as e: + DLOG.exception("Caught exception trying to delete host %s agent: %s" % + (host_name, e)) + return False + + if num_agents_found != len(supported_agents): + DLOG.warn("Host %s, did not find expected agents to delete" + % host_name) + + return True + + def delete_host_services_by_name(token, host_name, host_uuid, only_if_changed=False): """ @@ -437,6 +496,51 @@ def enable_host_services(token, host_uuid): return response +def enable_network_agents(token, host_name): + """ + Asks OpenStack Neutron to enable agents on a host + """ + try: + url, api_cmd, api_cmd_headers, result_data = get_network_agents( + token, host_name) + + payload = dict() + payload['admin_state_up'] = True + api_cmd_payload = dict() + api_cmd_payload['agent'] = payload + + num_agents_found = 0 + all_enabled = True + supported_agents = [AGENT_TYPE.L3, AGENT_TYPE.DHCP] + for agent in result_data: + if (agent['host'] == host_name and + agent['agent_type'] in supported_agents): + alive = agent.get('alive', False) + if not alive: + DLOG.verbose("Host %s agent %s not yet alive", + (host_name, agent)) + return False + api_cmd = url + "/v2.0/agents/%s" % agent['id'] + response = rest_api_request(token, "PUT", api_cmd, + api_cmd_headers, + json.dumps(api_cmd_payload)) + all_enabled = all_enabled and \ + response.result_data['agent']['admin_state_up'] + num_agents_found = num_agents_found + 1 + + except Exception as e: + DLOG.exception("Caught exception trying to enable host %s agents: %s" % + (host_name, e)) + return False + + if num_agents_found != len(supported_agents): + DLOG.error("Host %s, did not find expected agents to enable" + % host_name) + return False + + return all_enabled + + def disable_host_services(token, host_uuid): """ Asks OpenStack Neutron to disable a host @@ -462,6 +566,46 @@ def disable_host_services(token, host_uuid): return response +def disable_network_agents(token, host_name): + """ + Asks OpenStack Neutron to disable agents on a host + """ + try: + url, api_cmd, api_cmd_headers, result_data = get_network_agents( + token, host_name) + + payload = dict() + payload['admin_state_up'] = False + api_cmd_payload = dict() + api_cmd_payload['agent'] = payload + + num_agents_found = 0 + all_disabled = True + supported_agents = [AGENT_TYPE.L3, AGENT_TYPE.DHCP] + for agent in result_data: + if (agent['host'] == host_name and + agent['agent_type'] in supported_agents): + api_cmd = url + "/v2.0/agents/%s" % agent['id'] + response = rest_api_request(token, "PUT", api_cmd, + api_cmd_headers, + json.dumps(api_cmd_payload)) + all_disabled = all_disabled and not \ + response.result_data['agent']['admin_state_up'] + num_agents_found = num_agents_found + 1 + + except Exception as e: + DLOG.exception("Caught exception trying to disable host %s agents: %s" % + (host_name, e)) + return False + + if num_agents_found != len(supported_agents): + DLOG.error("Host %s, did not find expected agents to disable" + % host_name) + return False + + return all_disabled + + def query_host_services(token, host_name): """ Asks OpenStack Neutron for the state of services on a host @@ -505,3 +649,55 @@ def query_host_services(token, host_name): host_state = 'down' return host_state + + +def query_network_agents(token, host_name, check_fully_up): + """ + Asks OpenStack Neutron for the state of agents on a host + """ + try: + url, api_cmd, api_cmd_headers, result_data = get_network_agents( + token, host_name) + + agent_state = 'up' + supported_agents = [AGENT_TYPE.L3, AGENT_TYPE.DHCP] + for supported_agent in supported_agents: + found = False + for agent in result_data: + agent_type = agent.get('agent_type', '') + host = agent.get('host', '') + if (agent_type == supported_agent) and (host == host_name): + DLOG.verbose("found agent %s for host %s" % + (supported_agent, host_name)) + alive = agent.get('alive', False) + admin_state_up = agent.get('admin_state_up', False) + # found the agent of interest. + found = True + break + if found: + if check_fully_up: + if not (alive and admin_state_up): + DLOG.verbose("host %s agent %s not fully up. alive: %s," + " admin_state_up: %s" % + (host_name, supported_agent, + alive, admin_state_up)) + agent_state = 'down' + break + else: + if not alive: + DLOG.verbose("host %s agent %s not alive" % + (host_name, supported_agent)) + agent_state = 'down' + break + else: + DLOG.error("host %s agent %s not present" % + (host_name, supported_agent)) + agent_state = 'down' + break + + except Exception as e: + DLOG.exception("Caught exception trying to query host %s " + "agent states: %s" % (host_name, e)) + agent_state = 'down' + + return agent_state diff --git a/nfv/nfv-vim/nfv_vim/host_fsm/_host_task_work.py b/nfv/nfv-vim/nfv_vim/host_fsm/_host_task_work.py index 6cda134a..b1513b14 100755 --- a/nfv/nfv-vim/nfv_vim/host_fsm/_host_task_work.py +++ b/nfv/nfv-vim/nfv_vim/host_fsm/_host_task_work.py @@ -710,6 +710,8 @@ class WaitHostServicesCreatedTaskWork(state_machine.StateTaskWork): """ Callback for wait host services created """ + from nfv_vim import objects + response = (yield) self._query_inprogress = False @@ -719,6 +721,12 @@ class WaitHostServicesCreatedTaskWork(state_machine.StateTaskWork): (self._service, self._host.name, response)) if response['completed']: + if (self._service == objects.HOST_SERVICES.NETWORK and + response['result-data'] != 'enabled'): + DLOG.verbose("Wait-Host-Services-Created callback for %s, " + "service %s failed" % (self._service, + self._host.name)) + return # A completed response means the service exists. self.task.task_work_complete( state_machine.STATE_TASK_WORK_RESULT.SUCCESS, @@ -741,6 +749,13 @@ class WaitHostServicesCreatedTaskWork(state_machine.StateTaskWork): nfvi.nfvi_query_compute_host_services( self._host.uuid, self._host.name, self._host.personality, self._callback()) + elif self._service == objects.HOST_SERVICES.NETWORK: + self._query_inprogress = True + check_fully_up = False + nfvi.nfvi_query_network_host_services( + self._host.uuid, self._host.name, self._host.personality, + check_fully_up, + self._callback()) else: reason = ("Trying to wait for unknown host service %s" % self._service) @@ -768,6 +783,13 @@ class WaitHostServicesCreatedTaskWork(state_machine.StateTaskWork): self._host.uuid, self._host.name, self._host.personality, self._callback()) + elif self._service == objects.HOST_SERVICES.NETWORK: + check_fully_up = False + nfvi.nfvi_query_network_host_services( + self._host.uuid, self._host.name, + self._host.personality, + check_fully_up, + self._callback()) handled = True return handled @@ -1210,8 +1232,10 @@ class AuditHostServicesTaskWork(state_machine.StateTaskWork): self._host.uuid, self._host.name, self._host.personality, self._callback()) elif self._service == objects.HOST_SERVICES.NETWORK: + check_fully_up = True nfvi.nfvi_query_network_host_services( self._host.uuid, self._host.name, self._host.personality, + check_fully_up, self._callback()) else: reason = ("Trying to query unknown " diff --git a/nfv/nfv-vim/nfv_vim/host_fsm/_host_tasks.py b/nfv/nfv-vim/nfv_vim/host_fsm/_host_tasks.py index 92831dc2..784ead3e 100755 --- a/nfv/nfv-vim/nfv_vim/host_fsm/_host_tasks.py +++ b/nfv/nfv-vim/nfv_vim/host_fsm/_host_tasks.py @@ -43,13 +43,14 @@ class AddHostTask(state_machine.StateTask): self._host_reference = weakref.ref(host) task_work_list = list() if not host.kubernetes_configured: - # We only create the compute service on non-kubernetes systems. + # We only create the compute and network services + # on non-kubernetes systems. if host.host_service_configured(objects.HOST_SERVICES.COMPUTE): task_work_list.append(CreateHostServicesTaskWork( self, host, objects.HOST_SERVICES.COMPUTE)) - if host.host_service_configured(objects.HOST_SERVICES.NETWORK): - task_work_list.append(CreateHostServicesTaskWork( - self, host, objects.HOST_SERVICES.NETWORK)) + if host.host_service_configured(objects.HOST_SERVICES.NETWORK): + task_work_list.append(CreateHostServicesTaskWork( + self, host, objects.HOST_SERVICES.NETWORK)) if host.host_service_configured(objects.HOST_SERVICES.GUEST): task_work_list.append(CreateHostServicesTaskWork( self, host, objects.HOST_SERVICES.GUEST)) @@ -159,6 +160,11 @@ class EnableHostTask(state_machine.StateTask): task_work_list.append(EnableHostServicesTaskWork( self, host, objects.HOST_SERVICES.COMPUTE)) if host.host_service_configured(objects.HOST_SERVICES.NETWORK): + if host.kubernetes_configured: + # In kubernetes systems we must wait for the network service + # to be created before enabling it. + task_work_list.append(WaitHostServicesCreatedTaskWork( + self, host, objects.HOST_SERVICES.NETWORK)) task_work_list.append(EnableHostServicesTaskWork( self, host, objects.HOST_SERVICES.NETWORK)) if host.host_service_configured(objects.HOST_SERVICES.GUEST): diff --git a/nfv/nfv-vim/nfv_vim/nfvi/_nfvi_network_module.py b/nfv/nfv-vim/nfv_vim/nfvi/_nfvi_network_module.py index 9551c46a..3e2898df 100755 --- a/nfv/nfv-vim/nfv_vim/nfvi/_nfvi_network_module.py +++ b/nfv/nfv-vim/nfv_vim/nfvi/_nfvi_network_module.py @@ -166,6 +166,7 @@ def nfvi_create_network_host_services(host_uuid, host_name, host_personality, def nfvi_query_network_host_services(host_uuid, host_name, host_personality, + check_fully_up, callback): """ Query network services @@ -173,6 +174,7 @@ def nfvi_query_network_host_services(host_uuid, host_name, host_personality, cmd_id = _network_plugin.invoke_plugin('query_host_services', host_uuid, host_name, host_personality, + check_fully_up, callback=callback) return cmd_id diff --git a/nfv/nfv-vim/nfv_vim/nfvi/api/v1/_nfvi_network_api.py b/nfv/nfv-vim/nfv_vim/nfvi/api/v1/_nfvi_network_api.py index 9827175b..0d345743 100755 --- a/nfv/nfv-vim/nfv_vim/nfvi/api/v1/_nfvi_network_api.py +++ b/nfv/nfv-vim/nfv_vim/nfvi/api/v1/_nfvi_network_api.py @@ -148,7 +148,8 @@ class NFVINetworkAPI(object): @abc.abstractmethod def query_host_services(self, future, host_uuid, host_name, - host_personality, callback): + host_personality, check_fully_up, + callback): """ Query network services on a host using the plugin """ diff --git a/nfv/nfv-vim/nfv_vim/objects/_host.py b/nfv/nfv-vim/nfv_vim/objects/_host.py index b82d7fee..4df54f9e 100755 --- a/nfv/nfv-vim/nfv_vim/objects/_host.py +++ b/nfv/nfv-vim/nfv_vim/objects/_host.py @@ -182,8 +182,7 @@ class Host(ObjectData): self._nfvi_host.openstack_compute) elif service == HOST_SERVICES.NETWORK: configured = (not nfvi.nfvi_network_plugin_disabled() and - (self._nfvi_host.openstack_compute or - self._nfvi_host.openstack_control)) + self._nfvi_host.openstack_compute) elif service == HOST_SERVICES.GUEST: configured = (not nfvi.nfvi_guest_plugin_disabled() and self._nfvi_host.openstack_compute)