Use FQDN for MGMT network

The management network is used extensively for all internal
communication.
Since the original use of the network was a private network before
it was exposed for external communication in a distributed cloud
configuration, it was never designed to be reconfigured.
To support MGMT network reconfiguration the idea is to configure the
applications to use the hostname/FQDN instead of a static MGMT IP
address.
In this way the MGMT network can be changed and the services and
applications will still work since they are using the hostname/FQDN
and the DNS will be responsible to translate to the current MGMT
IP address.
The use of FQDN will be applied for all installation modes: AIO-SX,
AIO-DX, Standard, AIO-PLUS and DC subclouds. But given the
complexities of supporting the multi-host reconfiguration,
the MGMT network reconfiguration will focus on support for AIO-SX
only.
The DNSMASQ service must start as soon as possible to translate
the FQDN to IP address.
Test plan ( Debian only )
 - AIO-SX and AIO-DX virtualbox installation IPv4/IPv6
 - Standard virtualbox installation IPv6
 - DC virtualbox installation IPv4 ( AIO-SX/DX subclouds )
 - AIO-SX and AIO-DX installation IPv4/IPv6
 - AIO-DX plus installation IPv6
 - DC IPv6 and subcloud AIO-SX
 - AIO-DX host-swact
 - DC IPv4 virtualbox with subcloud AIO-DX and AIO-DX
 - AIO-SX to AIO-DX migration
 - netstat -tupl ( no services are using the MGMT IP address )
 - Ran sanity/regression tests
 - Backup and Restore for AIO-SX/AIO-DX

Story: 2010722
Task: 48241

Change-Id: If340354755ec401dac1b0da2c93e278e390f81a9
Signed-off-by: Fabiano Correa Mercer <fabiano.correamercer@windriver.com>

sysinv/sysinv/sysinv/requirements.txt

@@ -5,6 +5,7 @@ amqplib>=0.6.1
 boto3
 botocore
 cryptography!=2.0  # BSD/Apache-2.0
+dnspython
 eventlet
 greenlet>=0.3.2 # MIT
 keyring

sysinv/sysinv/sysinv/sysinv/cert_mon/service.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2020-2022 Wind River Systems, Inc.
+# Copyright (c) 2020-2023 Wind River Systems, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -123,5 +123,5 @@ class CertificateMonitorService(service.Service):
 
     def subcloud_sysinv_endpoint_update(self, ctxt, subcloud_name, endpoint):
         """Update sysinv endpoint of dc token cache"""
-        LOG.info("Update subloud: %s sysinv endpoint" % subcloud_name)
+        LOG.info("Update subcloud: %s sysinv endpoint" % subcloud_name)
         self.manager.subcloud_sysinv_endpoint_update(subcloud_name, endpoint)

sysinv/sysinv/sysinv/sysinv/common/constants.py

@@ -196,6 +196,13 @@ STORAGE_1_HOSTNAME = '%s-1' % STORAGE_HOSTNAME
 STORAGE_2_HOSTNAME = '%s-2' % STORAGE_HOSTNAME
 # Other Storage Hostnames are built dynamically.
 
+# FQDN host entries
+INTERNAL_DOMAIN = 'internal'
+
+CONTROLLER_FQDN = '%s.%s' % (CONTROLLER_HOSTNAME, INTERNAL_DOMAIN)
+CONTROLLER_0_FQDN = '%s.%s' % (CONTROLLER_0_HOSTNAME, INTERNAL_DOMAIN)
+CONTROLLER_1_FQDN = '%s.%s' % (CONTROLLER_1_HOSTNAME, INTERNAL_DOMAIN)
+
 # Replication Peer groups
 PEER_PREFIX = 'group-'
 
@@ -2077,7 +2084,7 @@ DEFAULT_DNS_SERVICE_DOMAIN = 'cluster.local'
 
 # Ansible bootstrap
 ANSIBLE_BOOTSTRAP_FLAG = os.path.join(tsc.VOLATILE_PATH, ".ansible_bootstrap")
-ANSIBLE_BOOTSTRAP_COMPLETED_FLAG = os.path.join(tsc.CONFIG_PATH,
+ANSIBLE_BOOTSTRAP_COMPLETED_FLAG = os.path.join(tsc.PLATFORM_CONF_PATH,
                                                 ".bootstrap_completed")
 UNLOCK_READY_FLAG = os.path.join(tsc.PLATFORM_CONF_PATH, ".unlock_ready")
 INVENTORY_WAIT_TIMEOUT_IN_SECS = 120

sysinv/sysinv/sysinv/sysinv/common/utils.py

@@ -2718,6 +2718,18 @@ def is_inventory_config_complete(dbapi, forihostid):
         return False
 
 
+def is_fqdn_ready_to_use():
+    """
+    Return true if FQDN can be used instead of IP ADDRESS
+    The use of FQDN is limited to management network
+    after the bootstrap.
+    """
+    if (os.path.isfile(constants.ANSIBLE_BOOTSTRAP_COMPLETED_FLAG)):
+        return True
+
+    return False
+
+
 def is_std_system(dbapi):
     system = dbapi.isystem_get_one()
     return system.system_type == constants.TIS_STD_BUILD

sysinv/sysinv/sysinv/sysinv/common/wsgi_service.py

@@ -10,7 +10,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-# Copyright (c) 2017 Wind River Systems, Inc.
+# Copyright (c) 2017-2023 Wind River Systems, Inc.
 #
 
 import socket
@@ -23,6 +23,7 @@ from oslo_service import wsgi
 from sysinv._i18n import _
 from sysinv.api import app
 from sysinv.common import exception
+from sysinv.common import utils
 
 
 CONF = cfg.CONF
@@ -36,6 +37,9 @@ class WSGIService(service.ServiceBase):
         """Initialize, but do not start the WSGI server.
 
         :param name: The name of the WSGI server given to the loader.
+        :param host: The FQDN/IP WSGI will listen ( sysinv-api )
+        :param port: which port to listen
+        :param workers: list of workers to handle the requests
         :param use_ssl: Wraps the socket in an SSL context if True.
         :returns: None
         """
@@ -47,11 +51,14 @@ class WSGIService(service.ServiceBase):
                 _("api_workers value of %d is invalid, "
                   "must be greater than 0.") % self.workers)
 
-        socket_family = None
+        if not utils.is_valid_ip(host):
-        if IPAddress(host).version == 4:
+            socket_family = None
-            socket_family = socket.AF_INET
+        else:
-        elif IPAddress(host).version == 6:
+            socket_family = None
-            socket_family = socket.AF_INET6
+            if IPAddress(host).version == 4:
+                socket_family = socket.AF_INET
+            elif IPAddress(host).version == 6:
+                socket_family = socket.AF_INET6
 
         # If not defined, pool_size will default to 100. In order
         # to increase the amount of threads handling multiple parallel

sysinv/sysinv/sysinv/sysinv/conductor/manager.py

@@ -1177,17 +1177,19 @@ class ConductorManager(service.PeriodicService):
                 hostname = re.sub("-%s$" % constants.NETWORK_TYPE_MGMT,
                                   '', str(address.name))
 
-                if cutils.is_aio_simplex_system(self.dbapi):
+                if (hostname != constants.SYSTEM_CONTROLLER_GATEWAY_IP_NAME):
-                    hostname_internal = hostname + ".internal"
+                    controller_alias = [constants.CONTROLLER_HOSTNAME,
-                    controller_alias = ["registry.local", "controller-platform-nfs"]
+                                        constants.DOCKER_REGISTRY_HOST,
+                                        "controller-platform-nfs"]
 
-                    if not hostname_internal == "controller.internal":
+                    if hostname == constants.CONTROLLER_HOSTNAME:
                         addn_line_internal = self._dnsmasq_addn_host_entry_to_string(
-                                address.address, hostname_internal)
+                                address.address, constants.CONTROLLER_FQDN, controller_alias)
                     else:
+                        hostname_internal = hostname + "." + constants.INTERNAL_DOMAIN
+                        hostname_alias = [hostname]
                         addn_line_internal = self._dnsmasq_addn_host_entry_to_string(
-                                address.address, hostname_internal, controller_alias)
+                                address.address, hostname_internal, hostname_alias)
-
                     f_out_addn.write(addn_line_internal)
 
                 if address.interface:

sysinv/sysinv/sysinv/sysinv/puppet/barbican.py

@@ -6,6 +6,7 @@
 
 from sysinv.common import constants
 from sysinv.puppet import openstack
+from sysinv.common import utils
 
 
 class BarbicanPuppet(openstack.OpenstackBasePuppet):
@@ -42,6 +43,10 @@ class BarbicanPuppet(openstack.OpenstackBasePuppet):
     def get_system_config(self):
         ksuser = self._get_service_user_name(self.SERVICE_NAME)
 
+        host = (constants.CONTROLLER_FQDN
+                if utils.is_fqdn_ready_to_use()
+                else None)
+
         config = {
             'barbican::keystone::auth::public_url': self.get_public_url(),
             'barbican::keystone::auth::internal_url': self.get_internal_url(),
@@ -52,9 +57,9 @@ class BarbicanPuppet(openstack.OpenstackBasePuppet):
             'barbican::keystone::auth::configure_user_role': False,
 
             'barbican::keystone::authtoken::auth_url':
-                self._keystone_identity_uri(),
+                self._keystone_identity_uri(host),
             'barbican::keystone::authtoken::auth_uri':
-                self._keystone_auth_uri(),
+                self._keystone_auth_uri(host),
 
             'barbican::keystone::authtoken::user_domain_name':
                 self._get_service_user_domain_name(),

sysinv/sysinv/sysinv/sysinv/puppet/base.py

@@ -212,6 +212,19 @@ class BasePuppet(object):
         subnet = address.address + '/' + address.prefix
         return subnet
 
+    def _get_bind_host(self):
+        """
+        Retrieve bind host and host
+        The use of FQDN is limited to management network only.
+        """
+        if utils.is_fqdn_ready_to_use():
+            bind_host = constants.CONTROLLER_FQDN
+            host = constants.CONTROLLER_FQDN
+        else:
+            bind_host = self._get_management_address()
+            host = None
+        return bind_host, host
+
     def _get_subcloud_endpoint_address(self):
         try:
             address = self._format_url_address(self._get_admin_address())

sysinv/sysinv/sysinv/sysinv/puppet/certalarm.py

@@ -6,6 +6,7 @@
 
 from sysinv.puppet import openstack
 from sysinv.common import constants
+from sysinv.common import utils
 
 
 class CertAlarmPuppet(openstack.OpenstackBasePuppet):
@@ -24,11 +25,15 @@ class CertAlarmPuppet(openstack.OpenstackBasePuppet):
     def get_system_config(self):
         sysinv_user = self._get_service_user_name(self.SYSINV_SERVICE_NAME)
 
+        host = (constants.CONTROLLER_FQDN
+                if utils.is_fqdn_ready_to_use()
+                else None)
+
         config = {}
         config.update({
             # The auth info for local authentication
-            'sysinv::certalarm::local_keystone_auth_uri': self._keystone_auth_uri(),
+            'sysinv::certalarm::local_keystone_auth_uri': self._keystone_auth_uri(host),
-            'sysinv::certalarm::local_keystone_identity_uri': self._keystone_identity_uri(),
+            'sysinv::certalarm::local_keystone_identity_uri': self._keystone_identity_uri(host),
             'sysinv::certalarm::local_keystone_project_domain': self._get_service_project_domain_name(),
             'sysinv::certalarm::local_keystone_tenant': self._get_service_project_name(),
             'sysinv::certalarm::local_keystone_user': sysinv_user,
@@ -40,8 +45,8 @@ class CertAlarmPuppet(openstack.OpenstackBasePuppet):
             dc_user = self._get_service_user_name(self.DC_SERVICE_NAME),
             config.update({
                 # The auth info for DC authentication
-                'sysinv::certalarm::dc_keystone_auth_uri': self._keystone_auth_uri(),
+                'sysinv::certalarm::dc_keystone_auth_uri': self._keystone_auth_uri(host),
-                'sysinv::certalarm::dc_keystone_identity_uri': self._keystone_identity_uri(),
+                'sysinv::certalarm::dc_keystone_identity_uri': self._keystone_identity_uri(host),
                 'sysinv::certalarm::dc_keystone_project_domain': self._get_service_project_domain_name(),
                 'sysinv::certalarm::dc_keystone_tenant': self._get_service_project_name(),
                 'sysinv::certalarm::dc_keystone_user': dc_user,

sysinv/sysinv/sysinv/sysinv/puppet/certmon.py

@@ -6,6 +6,7 @@
 
 from sysinv.puppet import openstack
 from sysinv.common import constants
+from sysinv.common import utils
 
 
 class CertMonPuppet(openstack.OpenstackBasePuppet):
@@ -24,11 +25,15 @@ class CertMonPuppet(openstack.OpenstackBasePuppet):
     def get_system_config(self):
         sysinv_user = self._get_service_user_name(self.SYSINV_SERVICE_NAME)
 
+        host = (constants.CONTROLLER_FQDN
+                if utils.is_fqdn_ready_to_use()
+                else None)
+
         config = {}
         config.update({
             # The auth info for local authentication
-            'sysinv::certmon::local_keystone_auth_uri': self._keystone_auth_uri(),
+            'sysinv::certmon::local_keystone_auth_uri': self._keystone_auth_uri(host),
-            'sysinv::certmon::local_keystone_identity_uri': self._keystone_identity_uri(),
+            'sysinv::certmon::local_keystone_identity_uri': self._keystone_identity_uri(host),
             'sysinv::certmon::local_keystone_project_domain': self._get_service_project_domain_name(),
             'sysinv::certmon::local_keystone_tenant': self._get_service_project_name(),
             'sysinv::certmon::local_keystone_user': sysinv_user,
@@ -40,8 +45,8 @@ class CertMonPuppet(openstack.OpenstackBasePuppet):
             dc_user = self._get_service_user_name(self.DC_SERVICE_NAME),
             config.update({
                 # The auth info for DC authentication
-                'sysinv::certmon::dc_keystone_auth_uri': self._keystone_auth_uri(),
+                'sysinv::certmon::dc_keystone_auth_uri': self._keystone_auth_uri(host),
-                'sysinv::certmon::dc_keystone_identity_uri': self._keystone_identity_uri(),
+                'sysinv::certmon::dc_keystone_identity_uri': self._keystone_identity_uri(host),
                 'sysinv::certmon::dc_keystone_project_domain': self._get_service_project_domain_name(),
                 'sysinv::certmon::dc_keystone_tenant': self._get_service_project_name(),
                 'sysinv::certmon::dc_keystone_user': dc_user,

sysinv/sysinv/sysinv/sysinv/puppet/dcdbsync.py

@@ -44,6 +44,7 @@ class DCDBsyncPuppet(openstack.OpenstackBasePuppet):
 
     def get_system_config(self):
         ksuser = self._get_service_user_name(self.SERVICE_NAME)
+        bind_host, host = self._get_bind_host()
 
         config = {
             # The region in which the identity server can be found
@@ -58,10 +59,10 @@ class DCDBsyncPuppet(openstack.OpenstackBasePuppet):
                 self._get_service_user_domain_name(),
             'dcdbsync::keystone::auth::service_name': self.SERVICE_NAME,
             'dcdbsync::keystone::auth::tenant': self._get_service_tenant_name(),
-            'dcdbsync::api::bind_host': self._get_management_address(),
+            'dcdbsync::api::bind_host': bind_host,
-            'dcdbsync::api::keystone_auth_uri': self._keystone_auth_uri(),
+            'dcdbsync::api::keystone_auth_uri': self._keystone_auth_uri(host),
             'dcdbsync::api::keystone_identity_uri':
-                self._keystone_identity_uri(),
+                self._keystone_identity_uri(host),
             'dcdbsync::api::keystone_tenant': self._get_service_project_name(),
             'dcdbsync::api::keystone_user_domain':
                 self._get_service_user_domain_name(),

sysinv/sysinv/sysinv/sysinv/puppet/dcmanager.py

@@ -50,6 +50,7 @@ class DCManagerPuppet(openstack.OpenstackBasePuppet):
 
     def get_system_config(self):
         ksuser = self._get_service_user_name(self.SERVICE_NAME)
+        bind_host, host = self._get_bind_host()
 
         return {
             # The region in which the identity server can be found
@@ -68,10 +69,10 @@ class DCManagerPuppet(openstack.OpenstackBasePuppet):
                 self._operator.keystone.get_admin_project_name(),
             'dcmanager::keystone::auth::admin_project_domain':
                 self._operator.keystone.get_admin_project_domain(),
-            'dcmanager::api::bind_host': self._get_management_address(),
+            'dcmanager::api::bind_host': bind_host,
-            'dcmanager::api::keystone_auth_uri': self._keystone_auth_uri(),
+            'dcmanager::api::keystone_auth_uri': self._keystone_auth_uri(host),
             'dcmanager::api::keystone_identity_uri':
-                self._keystone_identity_uri(),
+                self._keystone_identity_uri(host),
             'dcmanager::api::keystone_tenant': self._get_service_project_name(),
             'dcmanager::api::keystone_user_domain':
                 self._get_service_user_domain_name(),

sysinv/sysinv/sysinv/sysinv/puppet/dcorch.py

@@ -76,6 +76,7 @@ class DCOrchPuppet(openstack.OpenstackBasePuppet):
     def get_system_config(self):
         ksuser = self._get_service_user_name(self.SERVICE_NAME)
         dm_ksuser = self._operator.dcmanager.get_ks_user_name()
+        bind_host, host = self._get_bind_host()
 
         config = {
             # The region in which the identity server can be found
@@ -157,10 +158,10 @@ class DCOrchPuppet(openstack.OpenstackBasePuppet):
             'dcorch::keystone::auth::service_name': self.SERVICE_NAME,
             'dcorch::keystone::auth::tenant': self._get_service_tenant_name(),
 
-            'dcorch::api_proxy::bind_host': self._get_management_address(),
+            'dcorch::api_proxy::bind_host': bind_host,
-            'dcorch::api_proxy::keystone_auth_uri': self._keystone_auth_uri(),
+            'dcorch::api_proxy::keystone_auth_uri': self._keystone_auth_uri(host),
             'dcorch::api_proxy::keystone_identity_uri':
-                self._keystone_identity_uri(),
+                self._keystone_identity_uri(host),
             'dcorch::api_proxy::keystone_tenant': self._get_service_project_name(),
             'dcorch::api_proxy::keystone_user_domain':
                 self._get_service_user_domain_name(),

sysinv/sysinv/sysinv/sysinv/puppet/fm.py

@@ -53,6 +53,10 @@ class FmPuppet(openstack.OpenstackBasePuppet):
             snmp_enabled_value = 0    # False
             snmp_trap_server_port = self.TRAP_SERVER_DEFAULT_PORT
 
+        host = (constants.CONTROLLER_FQDN
+                if utils.is_fqdn_ready_to_use()
+                else None)
+
         config = {
             'fm::keystone::auth::public_url': self.get_public_url(),
             'fm::keystone::auth::internal_url': self.get_internal_url(),
@@ -62,9 +66,9 @@ class FmPuppet(openstack.OpenstackBasePuppet):
             'fm::keystone::auth::tenant': self._get_service_tenant_name(),
 
             'fm::keystone::authtoken::auth_url':
-                self._keystone_identity_uri(),
+                self._keystone_identity_uri(host),
             'fm::keystone::authtoken::auth_uri':
-                self._keystone_auth_uri(),
+                self._keystone_auth_uri(host),
 
             'fm::keystone::authtoken::user_domain_name':
                 self._get_service_user_domain_name(),
@@ -77,7 +81,7 @@ class FmPuppet(openstack.OpenstackBasePuppet):
             'fm::keystone::authtoken::username': ksuser,
 
             'fm::auth::auth_url':
-                self._keystone_auth_uri(),
+                self._keystone_auth_uri(host),
             'fm::auth::auth_tenant_name':
                 self._get_service_tenant_name(),
 
@@ -102,8 +106,16 @@ class FmPuppet(openstack.OpenstackBasePuppet):
         return config
 
     def get_host_config(self, host):
+        if (utils.is_fqdn_ready_to_use() and
+                host.personality == constants.CONTROLLER):
+            if host.hostname == constants.CONTROLLER_0_HOSTNAME:
+                host_ip = constants.CONTROLLER_0_FQDN
+            if host.hostname == constants.CONTROLLER_1_HOSTNAME:
+                host_ip = constants.CONTROLLER_1_FQDN
+        else:
+            host_ip = host.mgmt_ip
         config = {
-            'platform::fm::params::api_host': host.mgmt_ip
+            'platform::fm::params::api_host': host_ip
         }
         return config
 

sysinv/sysinv/sysinv/sysinv/puppet/inventory.py

@@ -48,6 +48,7 @@ class SystemInventoryPuppet(openstack.OpenstackBasePuppet):
         neutron_region_name = self._get_service_region_name(constants.SERVICE_NAME_NEUTRON)
         nova_region_name = self._get_service_region_name(constants.SERVICE_NAME_NOVA)
         barbican_region_name = self._operator.barbican.get_region_name()
+        bind_host, host = self._get_bind_host()
 
         return {
             # The region in which the identity server can be found
@@ -64,11 +65,11 @@ class SystemInventoryPuppet(openstack.OpenstackBasePuppet):
             'sysinv::keystone::auth::service_name': self.SERVICE_NAME,
             'sysinv::keystone::auth::tenant': self._get_service_tenant_name(),
 
-            'sysinv::api::bind_host': self._get_management_address(),
+            'sysinv::api::bind_host': bind_host,
             'sysinv::api::pxeboot_host': self._get_pxeboot_address(),
-            'sysinv::api::keystone_auth_uri': self._keystone_auth_uri(),
+            'sysinv::api::keystone_auth_uri': self._keystone_auth_uri(host),
             'sysinv::api::keystone_identity_uri':
-                self._keystone_identity_uri(),
+                self._keystone_identity_uri(host),
             'sysinv::api::keystone_tenant': self._get_service_project_name(),
             'sysinv::api::keystone_user_domain':
                 self._get_service_user_domain_name(),
@@ -81,9 +82,9 @@ class SystemInventoryPuppet(openstack.OpenstackBasePuppet):
                 self._to_create_services(),
 
             'sysinv::api::openstack_keystone_auth_uri':
-                self._keystone_auth_uri(),
+                self._keystone_auth_uri(host),
             'sysinv::api::openstack_keystone_identity_uri':
-                self._keystone_identity_uri(),
+                self._keystone_identity_uri(host),
             'sysinv::api::openstack_keystone_user_domain':
                 self._operator.keystone.get_admin_user_domain(),
             'sysinv::api::openstack_keystone_project_domain':
@@ -95,7 +96,7 @@ class SystemInventoryPuppet(openstack.OpenstackBasePuppet):
             'sysinv::api::openstack_keyring_service':
                 self.OPENSTACK_KEYRING_SERVICE,
 
-            'sysinv::rpc_zeromq_conductor_bind_ip': self._get_management_address()
+            'sysinv::rpc_zeromq_conductor_bind_ip': bind_host
         }
 
     def get_host_config(self, host):

sysinv/sysinv/sysinv/sysinv/puppet/keystone.py

@@ -279,13 +279,17 @@ class KeystonePuppet(openstack.OpenstackBasePuppet):
     def get_auth_port(self):
         return self.SERVICE_PORT
 
-    def get_auth_uri(self):
+    def get_auth_uri(self, host=None):
         if self._region_config():
             service_config = self._get_service_config(self.SERVICE_NAME)
             return service_config.capabilities.get('auth_uri')
         else:
-            return "http://%s:5000" % self._format_url_address(
+            if(host is not None):
-                self._get_management_address())
+                # FQDN
+                return "http://%s:%s" % (host, self.SERVICE_PORT)
+            else:
+                return "http://%s:%s" % (self._format_url_address(
+                    self._get_management_address()), self.SERVICE_PORT)
 
     def get_openstack_auth_uri(self):
         location = self._get_service_default_dns_name(
@@ -295,13 +299,21 @@ class KeystonePuppet(openstack.OpenstackBasePuppet):
                               location)
         return url
 
-    def get_identity_uri(self):
+    def get_identity_uri(self, host=None):
         if self._region_config():
             service_config = self._get_service_config(self.SERVICE_NAME)
+            if(host is not None):
+                # FQDN
+                return "http://%s:%s" % (host, self.SERVICE_PORT)
+
             return service_config.capabilities.get('auth_url')
         else:
-            return "http://%s:%s" % (self._format_url_address(
+            if(host is not None):
-                self._get_management_address()), self.SERVICE_PORT)
+                # FQDN
+                return "http://%s:%s" % (host, self.SERVICE_PORT)
+            else:
+                return "http://%s:%s" % (self._format_url_address(
+                    self._get_management_address()), self.SERVICE_PORT)
 
     def get_auth_url(self):
         if self._region_config():

sysinv/sysinv/sysinv/sysinv/puppet/mtce.py

@@ -7,6 +7,7 @@
 from tsconfig.tsconfig import KEYRING_PATH
 from sysinv.common import constants
 from sysinv.puppet import openstack
+from sysinv.common import utils
 
 
 class MtcePuppet(openstack.OpenstackBasePuppet):
@@ -31,13 +32,17 @@ class MtcePuppet(openstack.OpenstackBasePuppet):
             constants.MTCE_MULTICAST_MGMT_IP_NAME,
             constants.NETWORK_TYPE_MULTICAST)
 
+        host = (constants.CONTROLLER_FQDN
+                if utils.is_fqdn_ready_to_use()
+                else None)
+
         config = {
             'platform::mtce::params::auth_host':
                 self._keystone_auth_address(),
             'platform::mtce::params::auth_port':
                 self._keystone_auth_port(),
             'platform::mtce::params::auth_uri':
-                self._keystone_auth_uri(),
+                self._keystone_auth_uri(host),
             'platform::mtce::params::auth_username':
                 self._get_service_user_name(self.SERVICE_NAME),
             'platform::mtce::params::auth_user_domain':

sysinv/sysinv/sysinv/sysinv/puppet/networking.py

@@ -60,6 +60,17 @@ class NetworkingPuppet(base.BasePuppet):
                 gateway_address,
         })
 
+        # create flag for the mate controller to use FQDN or not
+        if utils.is_fqdn_ready_to_use():
+            fqdn_ready = True
+        else:
+            fqdn_ready = False
+
+        config.update({
+            'platform::network::%s::params::fqdn_ready' % networktype:
+                fqdn_ready,
+        })
+
         # TODO(fcorream): platform-nfs-iaddress is just necessary to allow
         # an upgrade from StarlingX releases 6 or 7 to new releases.
         # remove it when StarlingX rel. 6 or 7 are not being used anymore

sysinv/sysinv/sysinv/sysinv/puppet/openstack.py

@@ -152,11 +152,11 @@ class OpenstackBasePuppet(base.BasePuppet):
     def _keystone_auth_port(self):
         return self._operator.keystone.get_auth_port()
 
-    def _keystone_auth_uri(self):
+    def _keystone_auth_uri(self, host=None):
-        return self._operator.keystone.get_auth_uri()
+        return self._operator.keystone.get_auth_uri(host)
 
-    def _keystone_identity_uri(self):
+    def _keystone_identity_uri(self, host=None):
-        return self._operator.keystone.get_identity_uri()
+        return self._operator.keystone.get_identity_uri(host)
 
     def _keystone_region_name(self):
         return self._operator.keystone._identity_specific_region_name()
@@ -229,7 +229,10 @@ class OpenstackBasePuppet(base.BasePuppet):
     def _format_database_connection(self, service,
                                     address=None, database=None):
         if not address:
-            address = self._get_management_address()
+            if cutils.is_fqdn_ready_to_use():
+                address = constants.CONTROLLER_FQDN
+            else:
+                address = self._get_management_address()
 
         if not database:
             database = service

sysinv/sysinv/sysinv/sysinv/puppet/patching.py

@@ -5,7 +5,6 @@
 #
 
 from sysinv.common import constants
-
 from sysinv.puppet import openstack
 
 
@@ -34,9 +33,11 @@ class PatchingPuppet(openstack.OpenstackBasePuppet):
         }
 
     def get_system_config(self):
+        bind_host, host = self._get_bind_host()
+
         ksuser = self._get_service_user_name(self.SERVICE_NAME)
-        patch_keystone_auth_uri = self._keystone_auth_uri()
+        patch_keystone_auth_uri = self._keystone_auth_uri(host)
-        patch_keystone_identity_uri = self._keystone_identity_uri()
+        patch_keystone_identity_uri = self._keystone_identity_uri(host)
         controller_multicast = self._get_address_by_name(
             constants.PATCH_CONTROLLER_MULTICAST_MGMT_IP_NAME,
             constants.NETWORK_TYPE_MULTICAST)
@@ -56,8 +57,7 @@ class PatchingPuppet(openstack.OpenstackBasePuppet):
                 self._get_service_user_domain_name(),
             'patching::api::keystone_project_domain':
                 self._get_service_project_domain_name(),
-            'patching::api::bind_host':
+            'patching::api::bind_host': bind_host,
-                self._get_management_address(),
 
             'patching::keystone::auth::public_url': self.get_public_url(),
             'patching::keystone::auth::internal_url': self.get_internal_url(),
@@ -69,9 +69,9 @@ class PatchingPuppet(openstack.OpenstackBasePuppet):
             'patching::keystone::auth::tenant': self._get_service_tenant_name(),
 
             'patching::keystone::authtoken::auth_url':
-                self._keystone_identity_uri(),
+                self._keystone_identity_uri(host),
             'patching::keystone::authtoken::auth_uri':
-                self._keystone_auth_uri(),
+                self._keystone_auth_uri(host),
 
             'patching::controller_multicast': controller_multicast.address,
             'patching::agent_multicast': agent_multicast.address,

sysinv/sysinv/sysinv/sysinv/puppet/platform.py

@@ -184,12 +184,20 @@ class PlatformPuppet(base.BasePuppet):
         }
 
     def _get_amqp_config(self):
-        return {
+        if utils.is_fqdn_ready_to_use():
-            'platform::amqp::params::host':
+            amqp_config_dict = {
-                self._get_management_address(),
+                'platform::amqp::params::host': constants.CONTROLLER_FQDN,
-            'platform::amqp::params::host_url':
+                'platform::amqp::params::host_url': constants.CONTROLLER_FQDN,
-                self._format_url_address(self._get_management_address()),
+            }
-        }
+        else:
+            amqp_config_dict = {
+                'platform::amqp::params::host':
+                    self._get_management_address(),
+                'platform::amqp::params::host_url':
+                    self._format_url_address(self._get_management_address()),
+            }
+
+        return amqp_config_dict
 
     def _get_resolv_config(self):
         servers = [self._get_management_address()]
@@ -227,13 +235,18 @@ class PlatformPuppet(base.BasePuppet):
         public_address_url = self._format_url_address(public_address.address)
         https_enabled = self._https_enabled()
 
+        if utils.is_fqdn_ready_to_use():
+            priv_addr = constants.CONTROLLER_FQDN
+        else:
+            priv_addr = private_address.address
+
         config = {
             'platform::haproxy::params::public_ip_address':
                 public_address.address,
             'platform::haproxy::params::public_address_url':
                 public_address_url,
             'platform::haproxy::params::private_ip_address':
-                private_address.address,
+                priv_addr,
             'platform::haproxy::params::private_dc_ip_address':
                 private_dc_address.address,
             'platform::haproxy::params::enable_https':
@@ -306,30 +319,23 @@ class PlatformPuppet(base.BasePuppet):
 
         if host.personality == constants.CONTROLLER:
 
-            controller0_address = self._get_address_by_name(
-                constants.CONTROLLER_0_HOSTNAME, constants.NETWORK_TYPE_MGMT)
-
-            controller1_address = self._get_address_by_name(
-                constants.CONTROLLER_1_HOSTNAME, constants.NETWORK_TYPE_MGMT)
-
             if host.hostname == constants.CONTROLLER_0_HOSTNAME:
                 mate_hostname = constants.CONTROLLER_1_HOSTNAME
-                mate_address = controller1_address
             else:
                 mate_hostname = constants.CONTROLLER_0_HOSTNAME
-                mate_address = controller0_address
 
             config.update({
-                'platform::params::controller_0_ipaddress':
+                'platform::params::controller_fqdn':
-                    controller0_address.address,
+                    constants.CONTROLLER_FQDN,
-                'platform::params::controller_1_ipaddress':
+                'platform::params::controller_0_fqdn':
-                    controller1_address.address,
+                    constants.CONTROLLER_0_FQDN,
+                'platform::params::controller_1_fqdn':
+                    constants.CONTROLLER_1_FQDN,
                 'platform::params::controller_0_hostname':
                     constants.CONTROLLER_0_HOSTNAME,
                 'platform::params::controller_1_hostname':
                     constants.CONTROLLER_1_HOSTNAME,
                 'platform::params::mate_hostname': mate_hostname,
-                'platform::params::mate_ipaddress': mate_address.address,
             })
 
         system = self._get_system()

sysinv/sysinv/sysinv/sysinv/puppet/smapi.py

@@ -36,9 +36,13 @@ class SmPuppet(openstack.OpenstackBasePuppet):
     def get_system_config(self):
         ksuser = self._get_service_user_name(self.SERVICE_NAME)
 
+        host = (constants.CONTROLLER_FQDN
+                if utils.is_fqdn_ready_to_use()
+                else None)
+
         config = {
             'smapi::keystone::authtoken::username': ksuser,
-            'smapi::keystone::authtoken::auth_url': self._keystone_identity_uri(),
+            'smapi::keystone::authtoken::auth_url': self._keystone_identity_uri(host),
             'smapi::keystone::auth::auth_name': ksuser,
             'smapi::keystone::auth::public_url': self.get_public_url(),
             'smapi::keystone::auth::region': self._region_name(),
@@ -47,7 +51,7 @@ class SmPuppet(openstack.OpenstackBasePuppet):
 
             'platform::smapi::params::admin_url': self.get_admin_url(),
             'platform::smapi::params::internal_url': self.get_internal_url(),
-            'platform::smapi::params::keystone_auth_url': self._keystone_identity_uri(),
+            'platform::smapi::params::keystone_auth_url': self._keystone_identity_uri(host),
             'platform::smapi::params::keystone_username': ksuser,
             'platform::smapi::params::public_url': self.get_public_url(),
             'platform::smapi::params::port': self.SERVICE_PORT,
@@ -60,8 +64,17 @@ class SmPuppet(openstack.OpenstackBasePuppet):
         if (constants.CONTROLLER not in utils.get_personalities(host)):
             return {}
 
+        if (utils.is_fqdn_ready_to_use() and
+                host.personality == constants.CONTROLLER):
+            if host.hostname == constants.CONTROLLER_0_HOSTNAME:
+                host_ip = constants.CONTROLLER_0_FQDN
+            elif host.hostname == constants.CONTROLLER_1_HOSTNAME:
+                host_ip = constants.CONTROLLER_1_FQDN
+        else:
+            host_ip = host.mgmt_ip
+
         config = {
-            'platform::smapi::params::bind_ip': host.mgmt_ip,
+            'platform::smapi::params::bind_ip': host_ip,
         }
 
         return config

sysinv/sysinv/sysinv/sysinv/zmq_rpc/zmq_rpc.py

@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+import dns.resolver
 import zerorpc
 import eventlet
 import os
@@ -11,6 +12,8 @@ from oslo_log import log
 from zerorpc import exceptions
 
 from sysinv.db import api
+from sysinv.common import constants
+from sysinv.common import utils
 from sysinv.objects.base import SysinvObject
 from sysinv.zmq_rpc.client_provider import ClientProvider
 from sysinv.zmq_rpc.serializer import decode
@@ -81,12 +84,35 @@ class RpcWrapper(object):
 class ZmqRpcServer(object):
     def __init__(self, target, host, port):
         self.target = target
+        self.host = host
+        self.port = port
         self.endpoint = get_tcp_endpoint(host, port)
         self.server = None
 
     def run(self):
         def _run_in_thread():
             try:
+                if self.host in [constants.CONTROLLER_FQDN,
+                                 constants.CONTROLLER_0_FQDN,
+                                 constants.CONTROLLER_1_FQDN]:
+                    v4_results = dns.resolver.resolve(self.host, 'A',
+                                                      raise_on_no_answer=False)
+                    v6_results = dns.resolver.resolve(self.host, 'AAAA',
+                                                      raise_on_no_answer=False)
+                    dns_results = None
+                    if v6_results:
+                        dns_results = v6_results
+                    elif v4_results:
+                        dns_results = v4_results
+                    if dns_results is not None:
+                        host_ip = dns_results[0]
+                    else:
+                        LOG.error("Failed to resolve DNS host {}".format(self.host))
+                        return
+                    self.endpoint = get_tcp_endpoint(host_ip, self.port)
+                    LOG.debug("Resolved fqdn host={} host_ip={} endpoint={}"
+                              .format(self.host, host_ip, self.endpoint))
+
                 LOG.info("Starting zmq server at {}".format(self.endpoint))
                 # pylint: disable=unexpected-keyword-arg
                 # TODO with the default of 5s hearbeat we get LostRemote
@@ -138,7 +164,18 @@ class ZmqRpcClient(object):
                 raise Exception("Missing host_uuid parameter for rpc endpoint")
             dbapi = api.get_instance()
             host = dbapi.ihost_get(host_uuid)
-            endpoint = get_tcp_endpoint(host.mgmt_ip, self.port)
+            if (utils.is_fqdn_ready_to_use()
+                    and host.personality == constants.CONTROLLER):
+                if host.hostname == constants.CONTROLLER_0_HOSTNAME:
+                    host_fqdn = constants.CONTROLLER_0_FQDN
+                elif host.hostname == constants.CONTROLLER_1_HOSTNAME:
+                    host_fqdn = constants.CONTROLLER_1_FQDN
+                endpoint = get_tcp_endpoint(host_fqdn, self.port)
+            else:
+                address_name = utils.format_address_name(host.hostname,
+                                              constants.NETWORK_TYPE_MGMT)
+                address = dbapi.address_get_by_name(address_name)
+                endpoint = get_tcp_endpoint(address.address, self.port)
             client = client_provider.get_client_for_endpoint(endpoint)
 
         try:
@@ -193,7 +230,19 @@ class ZmqRpcClient(object):
                 "personality={}, subfunctions={})".format(
                     host.hostname, host.availability, host.operational,
                     host.personality, host.subfunctions))
-            endpoint = get_tcp_endpoint(host.mgmt_ip, self.port)
+            if (utils.is_initial_config_complete() and utils.is_fqdn_ready_to_use()
+                    and host.personality == constants.CONTROLLER):
+                if host.hostname == constants.CONTROLLER_0_HOSTNAME:
+                    host_fqdn = constants.CONTROLLER_0_FQDN
+                elif host.hostname == constants.CONTROLLER_1_HOSTNAME:
+                    host_fqdn = constants.CONTROLLER_1_FQDN
+                endpoint = get_tcp_endpoint(host_fqdn, self.port)
+            else:
+                address_name = utils.format_address_name(host.hostname,
+                                                         constants.NETWORK_TYPE_MGMT)
+                address = dbapi.address_get_by_name(address_name)
+                endpoint = get_tcp_endpoint(address.address, self.port)
+
             endpoints.append(endpoint)
             LOG.debug("Add host {} with endpoint {} to fanout request".format(
                 host.hostname, endpoint))