From 01da49ab8f69638d8bc35526a72ea3716237b8b0 Mon Sep 17 00:00:00 2001 From: Kevin Smith Date: Thu, 17 Jan 2019 07:02:46 -0500 Subject: [PATCH] Kubernetes Neutron VIM Host Management Refactoring Rework neutron system host management to operate on agent states rather than an extended host entity, as it was agreed with the neutron team that a new host level entity was not desired in the neutron core. DHCP and L3 agents on hosts with openstack_compute labels are now managed by the VIM. Story: 2003857 Task: 26663 Change-Id: I441fcf3c186f68d17abafe337af71d0caf9c40da Signed-off-by: Kevin Smith --- .../nfvi_plugins/nfvi_network_api.py | 126 ++++++++--- .../nfvi_plugins/openstack/neutron.py | 196 ++++++++++++++++++ .../nfv_vim/host_fsm/_host_task_work.py | 24 +++ nfv/nfv-vim/nfv_vim/host_fsm/_host_tasks.py | 14 +- .../nfv_vim/nfvi/_nfvi_network_module.py | 2 + .../nfv_vim/nfvi/api/v1/_nfvi_network_api.py | 3 +- nfv/nfv-vim/nfv_vim/objects/_host.py | 3 +- 7 files changed, 331 insertions(+), 37 deletions(-) diff --git a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_network_api.py b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_network_api.py index b1a0a70c..97a04b3f 100755 --- a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_network_api.py +++ b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_network_api.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: Apache-2.0 # +import os from six.moves import http_client as httplib from nfv_common import debug @@ -91,6 +92,15 @@ class NFVINetworkAPI(nfvi.api.v1.NFVINetworkAPI): (self._directory.get_service_info( OPENSTACK_SERVICE.NEUTRON) is not None)) + @staticmethod + def _host_supports_kubernetes(): + # TODO(ksmith): This check will disappear once kubernetes is the + # default + # Note we do not check personality as the assumption is that + # in the case of kubernetes, we won't even get to this check + # unless the correct label is applied to the host. + return (os.path.isfile('/etc/kubernetes/admin.conf')) + def get_networks(self, future, paging, callback): """ Get a list of networks @@ -775,6 +785,7 @@ class NFVINetworkAPI(nfvi.api.v1.NFVINetworkAPI): host_personality, callback): """ Create Host Services, notify neutron to create services for a host. + Only used in non k8s config. """ response = dict() response['completed'] = False @@ -940,14 +951,23 @@ class NFVINetworkAPI(nfvi.api.v1.NFVINetworkAPI): self._neutron_extensions = future.result.data - if neutron.lookup_extension(neutron.EXTENSION_NAMES.HOST, + if self._host_supports_kubernetes(): + extension_name = neutron.EXTENSION_NAMES.AGENT + else: + extension_name = neutron.EXTENSION_NAMES.HOST + + if neutron.lookup_extension(extension_name, self._neutron_extensions): response['reason'] = 'failed to delete neutron services' # Send the delete request to Neutron. - future.work(neutron.delete_host_services, - self._token, host_uuid) + if not self._host_supports_kubernetes(): + future.work(neutron.delete_host_services, + self._token, host_uuid) + else: + future.work(neutron.delete_network_agents, + self._token, host_name) try: future.result = (yield) @@ -1025,13 +1045,23 @@ class NFVINetworkAPI(nfvi.api.v1.NFVINetworkAPI): self._neutron_extensions = future.result.data - if neutron.lookup_extension(neutron.EXTENSION_NAMES.HOST, + if self._host_supports_kubernetes(): + extension_name = neutron.EXTENSION_NAMES.AGENT + else: + extension_name = neutron.EXTENSION_NAMES.HOST + + if neutron.lookup_extension(extension_name, self._neutron_extensions): response['reason'] = 'failed to enable neutron services' # Send the Enable request to Neutron - future.work(neutron.enable_host_services, - self._token, host_uuid) + if not self._host_supports_kubernetes(): + future.work(neutron.enable_host_services, + self._token, host_uuid) + else: + future.work(neutron.enable_network_agents, + self._token, host_name) + future.result = (yield) if not future.result.is_complete(): @@ -1041,14 +1071,21 @@ class NFVINetworkAPI(nfvi.api.v1.NFVINetworkAPI): % (host_uuid, host_name)) return - result_data = future.result.data['host'] - if not ('up' == result_data['availability'] and - host_name == result_data['name'] and - host_uuid == result_data['id']): - DLOG.error("Neutron enable-host-services failed, " - "operation did not complete, host_uuid=%s, " - "host_name=%s." % (host_uuid, host_name)) - return + if not self._host_supports_kubernetes(): + result_data = future.result.data['host'] + if not ('up' == result_data['availability'] and + host_name == result_data['name'] and + host_uuid == result_data['id']): + DLOG.error("Neutron enable-host-services failed, " + "operation did not complete, host_uuid=%s, " + "host_name=%s." % (host_uuid, host_name)) + return + else: + if not future.result.data: + DLOG.error("Neutron enable-host-services (agents) failed, " + "operation did not complete, host_uuid=%s, " + "host_name=%s." % (host_uuid, host_name)) + return response['completed'] = True response['reason'] = '' @@ -1073,7 +1110,8 @@ class NFVINetworkAPI(nfvi.api.v1.NFVINetworkAPI): callback.close() def query_host_services(self, future, host_uuid, host_name, - host_personality, callback): + host_personality, check_fully_up, + callback): """ Query Neutron Services for a host. """ @@ -1110,11 +1148,22 @@ class NFVINetworkAPI(nfvi.api.v1.NFVINetworkAPI): self._neutron_extensions = future.result.data - if neutron.lookup_extension(neutron.EXTENSION_NAMES.HOST, + if self._host_supports_kubernetes(): + extension_name = neutron.EXTENSION_NAMES.AGENT + else: + extension_name = neutron.EXTENSION_NAMES.HOST + + if neutron.lookup_extension(extension_name, self._neutron_extensions): # Send Query request to Neutron - future.work(neutron.query_host_services, - self._token, host_name) + if not self._host_supports_kubernetes(): + future.work(neutron.query_host_services, + self._token, host_name) + else: + future.work(neutron.query_network_agents, + self._token, host_name, + check_fully_up) + future.result = (yield) if not future.result.is_complete(): @@ -1189,14 +1238,24 @@ class NFVINetworkAPI(nfvi.api.v1.NFVINetworkAPI): self._neutron_extensions = future.result.data - if neutron.lookup_extension(neutron.EXTENSION_NAMES.HOST, + if self._host_supports_kubernetes(): + extension_name = neutron.EXTENSION_NAMES.AGENT + else: + extension_name = neutron.EXTENSION_NAMES.HOST + + if neutron.lookup_extension(extension_name, self._neutron_extensions): response['reason'] = 'failed to disable neutron services' # Send the Disable request to Neutron - future.work(neutron.disable_host_services, - self._token, - host_uuid) + if not self._host_supports_kubernetes(): + future.work(neutron.disable_host_services, + self._token, + host_uuid) + else: + future.work(neutron.disable_network_agents, + self._token, host_name) + future.result = (yield) if not future.result.is_complete(): @@ -1205,14 +1264,21 @@ class NFVINetworkAPI(nfvi.api.v1.NFVINetworkAPI): "host_name=%s." % (host_uuid, host_name)) return - result_data = future.result.data['host'] - if not ('down' == result_data['availability'] and - host_name == result_data['name'] and - host_uuid == result_data['id']): - DLOG.error("Neutron disable-host-services failed, " - "operation did not complete, host_uuid=%s, " - "host_name=%s." % (host_uuid, host_name)) - return + if not self._host_supports_kubernetes(): + result_data = future.result.data['host'] + if not ('down' == result_data['availability'] and + host_name == result_data['name'] and + host_uuid == result_data['id']): + DLOG.error("Neutron disable-host-services failed, " + "operation did not complete, host_uuid=%s, " + "host_name=%s." % (host_uuid, host_name)) + return + else: + if not future.result.data: + DLOG.error("Neutron disable-host-services (agents) failed, " + "operation did not complete, host_uuid=%s, " + "host_name=%s." % (host_uuid, host_name)) + return response['completed'] = True response['reason'] = '' diff --git a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/neutron.py b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/neutron.py index 1c4beb56..1d14fbd4 100755 --- a/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/neutron.py +++ b/nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/neutron.py @@ -24,6 +24,7 @@ class NeutronExtensionNames(Constants): Neutron Extension Name Constants """ HOST = Constant('host') + AGENT = Constant('agent') @six.add_metaclass(Singleton) @@ -46,10 +47,38 @@ class NetworkStatus(Constants): ERROR = Constant('ERROR') +@six.add_metaclass(Singleton) +class AgentType(Constants): + """ + AGENT TYPE Constants + """ + L3 = Constant('L3 agent') + DHCP = Constant('DHCP agent') + + # Constant Instantiation EXTENSION_NAMES = NeutronExtensionNames() NETWORK_ADMIN_STATE = NetworkAdministrativeState() NETWORK_STATUS = NetworkStatus() +AGENT_TYPE = AgentType() + + +def get_network_agents(token, host_name): + + url = token.get_service_url(OPENSTACK_SERVICE.NEUTRON) + if url is None: + raise ValueError("OpenStack Neutron URL is invalid") + + api_cmd = url + "/v2.0/agents?host=" + host_name + + api_cmd_headers = dict() + api_cmd_headers['wrs-header'] = 'true' + api_cmd_headers['Content-Type'] = "application/json" + + response = rest_api_request(token, "GET", api_cmd, api_cmd_headers) + result_data = response.result_data['agents'] + + return url, api_cmd, api_cmd_headers, result_data def lookup_extension(extension_name, extensions): @@ -385,6 +414,36 @@ def delete_host_services(token, host_uuid): return response +def delete_network_agents(token, host_name): + """ + Asks OpenStack Neutron to delete agents for a host + """ + try: + url, api_cmd, api_cmd_headers, result_data = get_network_agents( + token, host_name) + + num_agents_found = 0 + supported_agents = [AGENT_TYPE.L3, AGENT_TYPE.DHCP] + for agent in result_data: + if (agent['host'] == host_name and + agent['agent_type'] in supported_agents): + api_cmd = url + "/v2.0/agents/%s" % agent['id'] + response = rest_api_request(token, "DELETE", api_cmd, + api_cmd_headers) + num_agents_found = num_agents_found + 1 + + except Exception as e: + DLOG.exception("Caught exception trying to delete host %s agent: %s" % + (host_name, e)) + return False + + if num_agents_found != len(supported_agents): + DLOG.warn("Host %s, did not find expected agents to delete" + % host_name) + + return True + + def delete_host_services_by_name(token, host_name, host_uuid, only_if_changed=False): """ @@ -437,6 +496,51 @@ def enable_host_services(token, host_uuid): return response +def enable_network_agents(token, host_name): + """ + Asks OpenStack Neutron to enable agents on a host + """ + try: + url, api_cmd, api_cmd_headers, result_data = get_network_agents( + token, host_name) + + payload = dict() + payload['admin_state_up'] = True + api_cmd_payload = dict() + api_cmd_payload['agent'] = payload + + num_agents_found = 0 + all_enabled = True + supported_agents = [AGENT_TYPE.L3, AGENT_TYPE.DHCP] + for agent in result_data: + if (agent['host'] == host_name and + agent['agent_type'] in supported_agents): + alive = agent.get('alive', False) + if not alive: + DLOG.verbose("Host %s agent %s not yet alive", + (host_name, agent)) + return False + api_cmd = url + "/v2.0/agents/%s" % agent['id'] + response = rest_api_request(token, "PUT", api_cmd, + api_cmd_headers, + json.dumps(api_cmd_payload)) + all_enabled = all_enabled and \ + response.result_data['agent']['admin_state_up'] + num_agents_found = num_agents_found + 1 + + except Exception as e: + DLOG.exception("Caught exception trying to enable host %s agents: %s" % + (host_name, e)) + return False + + if num_agents_found != len(supported_agents): + DLOG.error("Host %s, did not find expected agents to enable" + % host_name) + return False + + return all_enabled + + def disable_host_services(token, host_uuid): """ Asks OpenStack Neutron to disable a host @@ -462,6 +566,46 @@ def disable_host_services(token, host_uuid): return response +def disable_network_agents(token, host_name): + """ + Asks OpenStack Neutron to disable agents on a host + """ + try: + url, api_cmd, api_cmd_headers, result_data = get_network_agents( + token, host_name) + + payload = dict() + payload['admin_state_up'] = False + api_cmd_payload = dict() + api_cmd_payload['agent'] = payload + + num_agents_found = 0 + all_disabled = True + supported_agents = [AGENT_TYPE.L3, AGENT_TYPE.DHCP] + for agent in result_data: + if (agent['host'] == host_name and + agent['agent_type'] in supported_agents): + api_cmd = url + "/v2.0/agents/%s" % agent['id'] + response = rest_api_request(token, "PUT", api_cmd, + api_cmd_headers, + json.dumps(api_cmd_payload)) + all_disabled = all_disabled and not \ + response.result_data['agent']['admin_state_up'] + num_agents_found = num_agents_found + 1 + + except Exception as e: + DLOG.exception("Caught exception trying to disable host %s agents: %s" % + (host_name, e)) + return False + + if num_agents_found != len(supported_agents): + DLOG.error("Host %s, did not find expected agents to disable" + % host_name) + return False + + return all_disabled + + def query_host_services(token, host_name): """ Asks OpenStack Neutron for the state of services on a host @@ -505,3 +649,55 @@ def query_host_services(token, host_name): host_state = 'down' return host_state + + +def query_network_agents(token, host_name, check_fully_up): + """ + Asks OpenStack Neutron for the state of agents on a host + """ + try: + url, api_cmd, api_cmd_headers, result_data = get_network_agents( + token, host_name) + + agent_state = 'up' + supported_agents = [AGENT_TYPE.L3, AGENT_TYPE.DHCP] + for supported_agent in supported_agents: + found = False + for agent in result_data: + agent_type = agent.get('agent_type', '') + host = agent.get('host', '') + if (agent_type == supported_agent) and (host == host_name): + DLOG.verbose("found agent %s for host %s" % + (supported_agent, host_name)) + alive = agent.get('alive', False) + admin_state_up = agent.get('admin_state_up', False) + # found the agent of interest. + found = True + break + if found: + if check_fully_up: + if not (alive and admin_state_up): + DLOG.verbose("host %s agent %s not fully up. alive: %s," + " admin_state_up: %s" % + (host_name, supported_agent, + alive, admin_state_up)) + agent_state = 'down' + break + else: + if not alive: + DLOG.verbose("host %s agent %s not alive" % + (host_name, supported_agent)) + agent_state = 'down' + break + else: + DLOG.error("host %s agent %s not present" % + (host_name, supported_agent)) + agent_state = 'down' + break + + except Exception as e: + DLOG.exception("Caught exception trying to query host %s " + "agent states: %s" % (host_name, e)) + agent_state = 'down' + + return agent_state diff --git a/nfv/nfv-vim/nfv_vim/host_fsm/_host_task_work.py b/nfv/nfv-vim/nfv_vim/host_fsm/_host_task_work.py index 6cda134a..b1513b14 100755 --- a/nfv/nfv-vim/nfv_vim/host_fsm/_host_task_work.py +++ b/nfv/nfv-vim/nfv_vim/host_fsm/_host_task_work.py @@ -710,6 +710,8 @@ class WaitHostServicesCreatedTaskWork(state_machine.StateTaskWork): """ Callback for wait host services created """ + from nfv_vim import objects + response = (yield) self._query_inprogress = False @@ -719,6 +721,12 @@ class WaitHostServicesCreatedTaskWork(state_machine.StateTaskWork): (self._service, self._host.name, response)) if response['completed']: + if (self._service == objects.HOST_SERVICES.NETWORK and + response['result-data'] != 'enabled'): + DLOG.verbose("Wait-Host-Services-Created callback for %s, " + "service %s failed" % (self._service, + self._host.name)) + return # A completed response means the service exists. self.task.task_work_complete( state_machine.STATE_TASK_WORK_RESULT.SUCCESS, @@ -741,6 +749,13 @@ class WaitHostServicesCreatedTaskWork(state_machine.StateTaskWork): nfvi.nfvi_query_compute_host_services( self._host.uuid, self._host.name, self._host.personality, self._callback()) + elif self._service == objects.HOST_SERVICES.NETWORK: + self._query_inprogress = True + check_fully_up = False + nfvi.nfvi_query_network_host_services( + self._host.uuid, self._host.name, self._host.personality, + check_fully_up, + self._callback()) else: reason = ("Trying to wait for unknown host service %s" % self._service) @@ -768,6 +783,13 @@ class WaitHostServicesCreatedTaskWork(state_machine.StateTaskWork): self._host.uuid, self._host.name, self._host.personality, self._callback()) + elif self._service == objects.HOST_SERVICES.NETWORK: + check_fully_up = False + nfvi.nfvi_query_network_host_services( + self._host.uuid, self._host.name, + self._host.personality, + check_fully_up, + self._callback()) handled = True return handled @@ -1210,8 +1232,10 @@ class AuditHostServicesTaskWork(state_machine.StateTaskWork): self._host.uuid, self._host.name, self._host.personality, self._callback()) elif self._service == objects.HOST_SERVICES.NETWORK: + check_fully_up = True nfvi.nfvi_query_network_host_services( self._host.uuid, self._host.name, self._host.personality, + check_fully_up, self._callback()) else: reason = ("Trying to query unknown " diff --git a/nfv/nfv-vim/nfv_vim/host_fsm/_host_tasks.py b/nfv/nfv-vim/nfv_vim/host_fsm/_host_tasks.py index 92831dc2..784ead3e 100755 --- a/nfv/nfv-vim/nfv_vim/host_fsm/_host_tasks.py +++ b/nfv/nfv-vim/nfv_vim/host_fsm/_host_tasks.py @@ -43,13 +43,14 @@ class AddHostTask(state_machine.StateTask): self._host_reference = weakref.ref(host) task_work_list = list() if not host.kubernetes_configured: - # We only create the compute service on non-kubernetes systems. + # We only create the compute and network services + # on non-kubernetes systems. if host.host_service_configured(objects.HOST_SERVICES.COMPUTE): task_work_list.append(CreateHostServicesTaskWork( self, host, objects.HOST_SERVICES.COMPUTE)) - if host.host_service_configured(objects.HOST_SERVICES.NETWORK): - task_work_list.append(CreateHostServicesTaskWork( - self, host, objects.HOST_SERVICES.NETWORK)) + if host.host_service_configured(objects.HOST_SERVICES.NETWORK): + task_work_list.append(CreateHostServicesTaskWork( + self, host, objects.HOST_SERVICES.NETWORK)) if host.host_service_configured(objects.HOST_SERVICES.GUEST): task_work_list.append(CreateHostServicesTaskWork( self, host, objects.HOST_SERVICES.GUEST)) @@ -159,6 +160,11 @@ class EnableHostTask(state_machine.StateTask): task_work_list.append(EnableHostServicesTaskWork( self, host, objects.HOST_SERVICES.COMPUTE)) if host.host_service_configured(objects.HOST_SERVICES.NETWORK): + if host.kubernetes_configured: + # In kubernetes systems we must wait for the network service + # to be created before enabling it. + task_work_list.append(WaitHostServicesCreatedTaskWork( + self, host, objects.HOST_SERVICES.NETWORK)) task_work_list.append(EnableHostServicesTaskWork( self, host, objects.HOST_SERVICES.NETWORK)) if host.host_service_configured(objects.HOST_SERVICES.GUEST): diff --git a/nfv/nfv-vim/nfv_vim/nfvi/_nfvi_network_module.py b/nfv/nfv-vim/nfv_vim/nfvi/_nfvi_network_module.py index 9551c46a..3e2898df 100755 --- a/nfv/nfv-vim/nfv_vim/nfvi/_nfvi_network_module.py +++ b/nfv/nfv-vim/nfv_vim/nfvi/_nfvi_network_module.py @@ -166,6 +166,7 @@ def nfvi_create_network_host_services(host_uuid, host_name, host_personality, def nfvi_query_network_host_services(host_uuid, host_name, host_personality, + check_fully_up, callback): """ Query network services @@ -173,6 +174,7 @@ def nfvi_query_network_host_services(host_uuid, host_name, host_personality, cmd_id = _network_plugin.invoke_plugin('query_host_services', host_uuid, host_name, host_personality, + check_fully_up, callback=callback) return cmd_id diff --git a/nfv/nfv-vim/nfv_vim/nfvi/api/v1/_nfvi_network_api.py b/nfv/nfv-vim/nfv_vim/nfvi/api/v1/_nfvi_network_api.py index 9827175b..0d345743 100755 --- a/nfv/nfv-vim/nfv_vim/nfvi/api/v1/_nfvi_network_api.py +++ b/nfv/nfv-vim/nfv_vim/nfvi/api/v1/_nfvi_network_api.py @@ -148,7 +148,8 @@ class NFVINetworkAPI(object): @abc.abstractmethod def query_host_services(self, future, host_uuid, host_name, - host_personality, callback): + host_personality, check_fully_up, + callback): """ Query network services on a host using the plugin """ diff --git a/nfv/nfv-vim/nfv_vim/objects/_host.py b/nfv/nfv-vim/nfv_vim/objects/_host.py index b82d7fee..4df54f9e 100755 --- a/nfv/nfv-vim/nfv_vim/objects/_host.py +++ b/nfv/nfv-vim/nfv_vim/objects/_host.py @@ -182,8 +182,7 @@ class Host(ObjectData): self._nfvi_host.openstack_compute) elif service == HOST_SERVICES.NETWORK: configured = (not nfvi.nfvi_network_plugin_disabled() and - (self._nfvi_host.openstack_compute or - self._nfvi_host.openstack_control)) + self._nfvi_host.openstack_compute) elif service == HOST_SERVICES.GUEST: configured = (not nfvi.nfvi_guest_plugin_disabled() and self._nfvi_host.openstack_compute)