From 949bab37d7d5e5a15aee3592d81a00b59ffec822 Mon Sep 17 00:00:00 2001 From: Teresa Ho Date: Fri, 21 Dec 2018 10:25:42 -0500 Subject: [PATCH] Kubernetes Cluster Network Configuration The Kubernetes cluster network is introduced and configurable. The cluster-host interface can be configured on any interface of the host and is defaulted to the management interface if it is not specified. The infrastructure network is no longer used in kubernetes config. SM and MTCE are setup to monitor the cluster-host if kubernetes is enabled. Nova live migration ip is set to use the cluster-host ip. Tests Performed: Containerized setup: AIO-SX: mgmt and cluster-host shared loopback interface AIO-DX: mgmt and cluster-host shared an interface AIO-DX: mgmt and cluster-host on different interface Standard 2+2+2: mgmt and cluster-host shared an interface Standard 2+2+2: mgmt and cluster-host on different interface For each of the setup, launch VM and connect to VM console Non-containerized deployments AIO-SX sanity AIO-DX sanity Standard 2+2 sanity Story: 2004273 Task: 27826 Change-Id: If6b918665131f01bc62687fbdc7978c5c103e3b7 Signed-off-by: Teresa Ho --- .../configutilities/common/configobjects.py | 5 +- .../configutilities/common/validator.py | 104 ++++ .../controllerconfig/configassistant.py | 579 +++++++++++++++++- .../tests/files/system_config.cluster | 59 ++ .../tests/files/system_config.lag.vlan | 7 + .../tests/test_system_config.py | 83 +++ .../src/modules/platform/manifests/config.pp | 9 +- .../src/modules/platform/manifests/dns.pp | 10 + .../modules/platform/manifests/kubernetes.pp | 15 +- .../src/modules/platform/manifests/network.pp | 19 + .../src/modules/platform/manifests/sm.pp | 77 ++- .../platform/templates/dnsmasq.conf.erb | 5 + .../platform/templates/kubeadm.yaml.erb | 3 +- .../sysinv/api/controllers/v1/address.py | 1 + .../sysinv/api/controllers/v1/address_pool.py | 1 - .../sysinv/api/controllers/v1/interface.py | 24 +- .../sysinv/api/controllers/v1/network.py | 16 +- .../sysinv/sysinv/sysinv/common/constants.py | 10 +- sysinv/sysinv/sysinv/sysinv/helm/nova.py | 34 +- sysinv/sysinv/sysinv/sysinv/puppet/base.py | 11 + .../sysinv/sysinv/sysinv/puppet/interface.py | 42 +- .../sysinv/sysinv/sysinv/puppet/kubernetes.py | 40 +- .../sysinv/sysinv/sysinv/puppet/networking.py | 17 + .../sysinv/sysinv/sysinv/puppet/platform.py | 17 + 24 files changed, 1101 insertions(+), 87 deletions(-) mode change 100755 => 100644 configutilities/configutilities/configutilities/common/configobjects.py mode change 100755 => 100644 configutilities/configutilities/configutilities/common/validator.py create mode 100755 controllerconfig/controllerconfig/controllerconfig/tests/files/system_config.cluster mode change 100755 => 100644 puppet-manifests/src/modules/platform/manifests/sm.pp diff --git a/configutilities/configutilities/configutilities/common/configobjects.py b/configutilities/configutilities/configutilities/common/configobjects.py old mode 100755 new mode 100644 index 679f583672..003d73db2a --- a/configutilities/configutilities/configutilities/common/configobjects.py +++ b/configutilities/configutilities/configutilities/common/configobjects.py @@ -20,9 +20,10 @@ SUBCLOUD_CONFIG = 2 MGMT_TYPE = 0 INFRA_TYPE = 1 OAM_TYPE = 2 +CLUSTER_TYPE = 3 NETWORK_PREFIX_NAMES = [ - ('MGMT', 'INFRA', 'OAM'), - ('CLM', 'BLS', 'CAN') + ('MGMT', 'INFRA', 'OAM', 'CLUSTER'), + ('CLM', 'BLS', 'CAN', 'CLUSTER') ] # Additions to this list must be reflected in the hostfile diff --git a/configutilities/configutilities/configutilities/common/validator.py b/configutilities/configutilities/configutilities/common/validator.py old mode 100755 new mode 100644 index afa404bfa2..ff0524c219 --- a/configutilities/configutilities/configutilities/common/validator.py +++ b/configutilities/configutilities/configutilities/common/validator.py @@ -14,6 +14,7 @@ from configutilities.common.configobjects import INFRA_TYPE from configutilities.common.configobjects import DEFAULT_DOMAIN_NAME from configutilities.common.configobjects import HP_NAMES from configutilities.common.configobjects import SUBCLOUD_CONFIG +from configutilities.common.configobjects import CLUSTER_TYPE from netaddr import IPRange from configutilities.common.utils import lag_mode_to_str from configutilities.common.utils import validate_network_str @@ -73,8 +74,10 @@ class ConfigValidator(object): self.pxeboot_section_name = None self.management_interface = None self.infrastructure_interface = None + self.cluster_interface = None self.mgmt_network = None self.infra_network = None + self.cluster_network = None self.oam_network = None self.vswitch_type = None self.glance_region = None @@ -780,6 +783,105 @@ class ConfigValidator(object): else: self.infrastructure_interface = "" + def validate_cluster(self): + # Kubernetes cluster network configuration + cluster_prefix = NETWORK_PREFIX_NAMES[self.naming_type][CLUSTER_TYPE] + if not self.conf.has_section(cluster_prefix + '_NETWORK'): + return + self.cluster_network = Network() + try: + self.cluster_network.parse_config(self.conf, self.config_type, + CLUSTER_TYPE, + min_addresses=8, + naming_type=self.naming_type) + except ConfigFail: + raise + except Exception as e: + raise ConfigFail("Error parsing configuration file: %s" % e) + + if self.cluster_network.floating_address: + raise ConfigFail("%s network cannot specify individual unit " + "addresses" % cluster_prefix) + + try: + check_network_overlap(self.cluster_network.cidr, + self.configured_networks) + self.configured_networks.append(self.cluster_network.cidr) + except ValidateFail: + raise ConfigFail("%s CIDR %s overlaps with another configured " + "network" % + (cluster_prefix, str(self.cluster_network.cidr))) + + if self.cluster_network.logical_interface.lag_interface: + supported_lag_mode = [1, 4] + if (self.cluster_network.logical_interface.lag_mode not in + supported_lag_mode): + raise ConfigFail( + "Unsupported LAG mode (%d) for %s interface" + " - use LAG mode %s instead" % + (self.cluster_network.logical_interface.lag_mode, + cluster_prefix, supported_lag_mode)) + + self.cluster_interface = 'bond' + str(self.next_lag_index) + cluster_interface_name = self.cluster_interface + self.next_lag_index += 1 + else: + self.cluster_interface = ( + self.cluster_network.logical_interface.ports[0]) + cluster_interface_name = self.cluster_interface + + if self.cluster_network.vlan: + if any(self.cluster_network.vlan == vlan for vlan in + self.configured_vlans): + raise ConfigFail( + "%s_NETWORK VLAN conflicts with another configured " + "VLAN" % cluster_prefix) + self.configured_vlans.append(self.cluster_network.vlan) + cluster_interface_name += '.' + str(self.cluster_network.vlan) + + mtu = self.cluster_network.logical_interface.mtu + if not is_mtu_valid(mtu): + raise ConfigFail( + "Invalid MTU value of %s for %s. " + "Valid values: 576 - 9216" + % (mtu, self.cluster_network.logical_interface.name)) + + if self.cgcs_conf is not None: + self.cgcs_conf.add_section('cCLUSTER') + self.cgcs_conf.set('cCLUSTER', 'CLUSTER_MTU', + self.cluster_network.logical_interface.mtu) + self.cgcs_conf.set('cCLUSTER', 'CLUSTER_SUBNET', + self.cluster_network.cidr) + if self.cluster_network.logical_interface.lag_interface: + self.cgcs_conf.set('cCLUSTER', 'LAG_CLUSTER_INTERFACE', 'yes') + self.cgcs_conf.set( + 'cCLUSTER', 'CLUSTER_BOND_MEMBER_0', + self.cluster_network.logical_interface.ports[0]) + self.cgcs_conf.set( + 'cCLUSTER', 'CLUSTER_BOND_MEMBER_1', + self.cluster_network.logical_interface.ports[1]) + self.cgcs_conf.set('cCLUSTER', 'CLUSTER_BOND_POLICY', + lag_mode_to_str(self.cluster_network. + logical_interface.lag_mode)) + else: + self.cgcs_conf.set('cCLUSTER', 'LAG_CLUSTER_INTERFACE', 'no') + self.cgcs_conf.set('cCLUSTER', 'CLUSTER_INTERFACE', + self.cluster_interface) + + if self.cluster_network.vlan: + self.cgcs_conf.set('cCLUSTER', 'CLUSTER_VLAN', + str(self.cluster_network.vlan)) + + self.cgcs_conf.set('cCLUSTER', 'CLUSTER_INTERFACE_NAME', + cluster_interface_name) + + if self.cluster_network.dynamic_allocation: + self.cgcs_conf.set('cCLUSTER', 'DYNAMIC_ADDRESS_ALLOCATION', + "yes") + else: + self.cgcs_conf.set('cCLUSTER', 'DYNAMIC_ADDRESS_ALLOCATION', + "no") + def validate_oam(self): # OAM network configuration oam_prefix = NETWORK_PREFIX_NAMES[self.naming_type][OAM_TYPE] @@ -1295,6 +1397,8 @@ def validate(system_config, config_type=REGION_CONFIG, cgcs_config=None, validator.validate_infra() # OAM network configuration validator.validate_oam() + # Kubernetes Cluster network configuration + validator.validate_cluster() # Neutron configuration - leave blank to use defaults # DNS configuration validator.validate_dns() diff --git a/controllerconfig/controllerconfig/controllerconfig/configassistant.py b/controllerconfig/controllerconfig/controllerconfig/configassistant.py index 43a204d2cf..ba93e26590 100644 --- a/controllerconfig/controllerconfig/controllerconfig/configassistant.py +++ b/controllerconfig/controllerconfig/controllerconfig/configassistant.py @@ -441,6 +441,26 @@ class ConfigAssistant(): self.external_oam_address_1 = IPAddress("10.10.10.4") self.oamcontroller_floating_hostname = "oamcontroller" + # Kubernetes cluster network config + self.cluster_host_interface_configured = False + self.cluster_host_interface_name = self.management_interface_name + self.cluster_host_interface = self.management_interface + self.cluster_host_vlan = "" + self.cluster_host_mtu = constants.LINK_MTU_DEFAULT + self.lag_cluster_host_interface = False + self.lag_cluster_host_interface_member0 = "" + self.lag_cluster_host_interface_member1 = "" + self.lag_cluster_host_interface_policy = \ + constants.LAG_MODE_ACTIVE_BACKUP + self.lag_cluster_host_interface_txhash = "" + self.lag_cluster_host_interface_miimon = \ + constants.LAG_MIIMON_FREQUENCY + self.cluster_host_subnet = IPNetwork("192.168.206.0/24") + + # Will be configurable in the future + self.cluster_pod_subnet = IPNetwork("172.16.0.0/16") + self.cluster_service_subnet = IPNetwork("10.96.0.0/12") + # SDN config self.enable_sdn = False # HTTPS @@ -740,7 +760,12 @@ class ConfigAssistant(): (self.external_oam_interface_configured and self.lag_external_oam_interface and (interface_name == self.lag_external_oam_interface_member0 or - interface_name == self.lag_external_oam_interface_member1))): + interface_name == self.lag_external_oam_interface_member1)) + or + (self.cluster_host_interface_configured and + self.lag_cluster_host_interface and + (interface_name == self.lag_cluster_host_interface_member0 or + interface_name == self.lag_cluster_host_interface_member1))): return True else: return False @@ -757,7 +782,9 @@ class ConfigAssistant(): (self.infrastructure_interface_configured and interface_name == self.infrastructure_interface) or (self.external_oam_interface_configured and - interface_name == self.external_oam_interface)): + interface_name == self.external_oam_interface) or + (self.cluster_host_interface_configured and + interface_name == self.cluster_host_interface)): return True else: return False @@ -1294,11 +1321,18 @@ class ConfigAssistant(): break while True: print('') - print(textwrap.fill( - "IP addresses can be assigned to hosts dynamically or " - "a static IP address can be specified for each host. " - "This choice applies to both the management network " - "and infrastructure network (if configured). ", 80)) + if self.kubernetes: + print(textwrap.fill( + "IP addresses can be assigned to hosts dynamically or " + "a static IP address can be specified for each host. " + "This choice applies to both the management network " + "and cluster-host network. ", 80)) + else: + print(textwrap.fill( + "IP addresses can be assigned to hosts dynamically or " + "a static IP address can be specified for each host. " + "This choice applies to both the management network " + "and infrastructure network (if configured). ", 80)) print(textwrap.fill( "Warning: Selecting 'N', or static IP address allocation, " "disables automatic provisioning of new hosts in System " @@ -2331,6 +2365,307 @@ class ConfigAssistant(): """ External OAM interface configuration complete""" self.external_oam_interface_configured = True + def is_valid_cluster_host_address(self, ip_address): + """Determine whether cluster host address is valid.""" + if ip_address == self.cluster_host_subnet.network: + print("Cannot use network address") + return False + elif ip_address == self.cluster_host_subnet.broadcast: + print("Cannot use broadcast address") + return False + elif ip_address.is_multicast(): + print("Invalid network address - multicast address not allowed") + return False + elif ip_address.is_loopback(): + print("Invalid network address - loopback address not allowed") + return False + elif ip_address not in self.cluster_host_subnet: + print("Address must be in the cluster host subnet") + return False + else: + return True + + def input_cluster_host_config(self): + """Allow user to input cluster-host config and perform validation.""" + + print("\nCluster Host Network:") + print("-----------------------\n") + + print((textwrap.fill( + "The cluster host network is used for internal communication " + "between Kubernetes clusters. " + "IP addresses on this network are reachable only within the data " + "center.", 80))) + print('') + print(textwrap.fill( + "If a separate cluster host interface is not configured the " + "management network will be used.", 80)) + print('') + + while True: + print('') + print(textwrap.fill( + "An cluster host bond interface provides redundant " + "connections for the cluster host network.", 80)) + print('') + user_input = input( + "Cluster host interface link aggregation [y/N]: ") + if user_input.lower() == 'q': + raise UserQuit + elif user_input.lower() == 'y': + self.lag_cluster_host_interface = True + break + elif user_input.lower() in ('n', ''): + self.lag_cluster_host_interface = False + break + else: + print("Invalid choice") + continue + + while True: + if self.lag_cluster_host_interface: + self.cluster_host_interface = self.get_next_lag_name() + + user_input = input("Cluster host interface [" + + str(self.management_interface) + "]: ") + if user_input.lower() == 'q': + raise UserQuit + elif user_input == '': + user_input = self.management_interface + if user_input == '': + print("Invalid interface") + continue + elif self.lag_cluster_host_interface: + print(textwrap.fill( + "Warning: The default name for the cluster host bond " + "interface (%s) cannot be changed." % + self.cluster_host_interface, 80)) + print('') + user_input = self.cluster_host_interface + + if self.is_interface_in_bond(user_input): + print(textwrap.fill( + "Interface is already configured as part of an " + "aggregated interface.", 80)) + continue + elif self.lag_cluster_host_interface: + self.cluster_host_interface = user_input + self.cluster_host_interface_name = user_input + break + elif interface_exists(user_input): + self.cluster_host_interface = user_input + self.cluster_host_interface_name = user_input + break + else: + print("Interface does not exist") + continue + + while True: + user_input = input( + "Configure an cluster host VLAN [y/N]: ") + if user_input.lower() == 'q': + raise UserQuit + elif user_input.lower() == 'y': + while True: + user_input = input( + "Cluster host VLAN Identifier [" + + str(self.cluster_host_vlan) + "]: ") + if user_input.lower() == 'q': + raise UserQuit + elif is_valid_vlan(user_input): + self.cluster_host_vlan = user_input + self.cluster_host_interface_name = \ + self.cluster_host_interface + '.' + \ + self.cluster_host_vlan + break + else: + print("VLAN is invalid/unsupported") + continue + break + elif user_input.lower() in ('n', ''): + self.cluster_host_vlan = "" + break + else: + print("Invalid choice") + continue + + while True: + if self.cluster_host_interface == self.management_interface: + break + user_input = input("Cluster host interface MTU [" + + str(self.cluster_host_mtu) + "]: ") + if user_input.lower() == 'q': + raise UserQuit + elif user_input == "": + user_input = self.cluster_host_mtu + + if (self.management_interface_configured and + self.cluster_host_interface == + self.management_interface and + self.cluster_host_vlan and + user_input > self.management_mtu): + print("Cluster host VLAN MTU must not be larger than " + "underlying management interface MTU") + continue + elif is_mtu_valid(user_input): + self.cluster_host_mtu = user_input + break + else: + print("MTU is invalid/unsupported") + continue + + while True: + if self.cluster_host_interface == self.management_interface: + break + if not self.lag_cluster_host_interface: + break + print('') + print("Specify one of the bonding policies. Possible values are:") + print(" 1) Active-backup policy") + print(" 2) Balanced XOR policy") + print(" 3) 802.3ad (LACP) policy") + + user_input = input( + "\nCluster host interface bonding policy [" + + str(self.lag_cluster_host_interface_policy) + "]: ") + if user_input.lower() == 'q': + raise UserQuit + elif user_input == '1': + self.lag_cluster_host_interface_policy = \ + constants.LAG_MODE_ACTIVE_BACKUP + self.lag_cluster_host_interface_txhash = None + break + elif user_input == '2': + self.lag_cluster_host_interface_policy = \ + constants.LAG_MODE_BALANCE_XOR + self.lag_cluster_host_interface_txhash = \ + constants.LAG_TXHASH_LAYER2 + break + elif user_input == '3': + self.lag_cluster_host_interface_policy = \ + constants.LAG_MODE_8023AD + self.lag_cluster_host_interface_txhash = \ + constants.LAG_TXHASH_LAYER2 + break + elif user_input == "": + break + else: + print("Invalid choice") + continue + + while True: + if not self.lag_cluster_host_interface: + break + if self.cluster_host_interface == self.management_interface: + break + + print(textwrap.fill( + "A maximum of 2 physical interfaces can be attached to the " + "cluster host interface.", 80)) + print('') + + user_input = input( + "First cluster host interface member [" + + str(self.lag_cluster_host_interface_member0) + "]: ") + if user_input.lower() == 'q': + raise UserQuit + elif user_input == "": + user_input = self.lag_cluster_host_interface_member0 + + if self.is_interface_in_bond(user_input): + print(textwrap.fill( + "Interface is already configured as part of an " + "aggregated interface.", 80)) + continue + elif self.is_interface_in_use(user_input): + print("Interface is already in use") + continue + elif interface_exists(user_input): + self.lag_cluster_host_interface_member0 = user_input + else: + print("Interface does not exist") + self.lag_cluster_host_interface_member0 = "" + continue + + user_input = input( + "Second cluster host interface member [" + + str(self.lag_cluster_host_interface_member1) + "]: ") + if user_input.lower() == 'q': + raise UserQuit + elif user_input == "": + user_input = self.lag_cluster_host_interface_member1 + + if self.is_interface_in_bond(user_input): + print(textwrap.fill( + "Interface is already configured as part of an " + "aggregated interface.", 80)) + continue + elif self.is_interface_in_use(user_input): + print("Interface is already in use") + continue + elif interface_exists(user_input): + if user_input == self.lag_cluster_host_interface_member0: + print("Cannot use member 0 as member 1") + continue + else: + self.lag_cluster_host_interface_member1 = user_input + break + else: + print("Interface does not exist") + self.lag_cluster_host_interface_member1 = "" + user_input = input( + "Do you want a single physical member in the bond " + "interface [y/n]: ") + if user_input.lower() == 'q': + raise UserQuit + elif user_input.lower() == 'y': + break + elif user_input.lower() in ('n', ''): + continue + else: + print("Invalid choice") + continue + + min_addresses = 8 + while True: + user_input = input("Cluster subnet [" + + str(self.cluster_host_subnet) + "]: ") + if user_input.lower() == 'q': + raise UserQuit + elif user_input == "": + user_input = self.cluster_host_subnet + + try: + ip_input = IPNetwork(user_input) + if ip_input.ip != ip_input.network: + print("Invalid network address") + continue + elif ip_input.size < min_addresses: + print("Cluster subnet too small - " + "must have at least 16 addresses") + continue + elif ip_input.version == 6 and ip_input.prefixlen < 64: + print("IPv6 minimum prefix length is 64") + continue + elif ((self.separate_pxeboot_network and + ip_input.ip in self.pxeboot_subnet) or + ip_input.ip in self.management_subnet): + print("Cluster host subnet overlaps with an already " + "configured subnet") + continue + + if ip_input.size < 255: + print("WARNING: Subnet allows only %d addresses." + % ip_input.size) + + self.cluster_host_subnet = ip_input + break + except AddrFormatError: + print("Invalid subnet - please enter a valid IPv4 subnet") + + """ Cluster host interface configuration complete""" + self.cluster_host_interface_configured = True + def input_authentication_config(self): """Allow user to input authentication config and perform validation. """ @@ -2375,7 +2710,6 @@ class ConfigAssistant(): self.admin_password = "Li69nux*" self.management_interface_configured = True self.external_oam_interface_configured = True - self.default_pxeboot_config() if utils.is_cpe(): @@ -2403,11 +2737,15 @@ class ConfigAssistant(): self.input_pxeboot_config() self.input_management_config() if self.system_dc_role != \ - sysinv_constants.DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER: + sysinv_constants.DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER \ + and not self.kubernetes: # Disallow infrastructure network on systemcontroller, # as services located on infrastructure network will not # be reachable by subclouds. + # Disallow infrastructure network on Kubernetes configs also. self.input_infrastructure_config() + if self.kubernetes: + self.input_cluster_host_config() self.input_external_oam_config() self.input_authentication_config() @@ -2667,6 +3005,32 @@ class ConfigAssistant(): not self.infrastructure_end_address: self.use_entire_infra_subnet = True + # Cluster network configuration + if self.kubernetes: + self.cluster_host_interface_name = config.get( + 'cCLUSTER', 'CLUSTER_INTERFACE_NAME') + self.cluster_host_interface = config.get( + 'cCLUSTER', 'CLUSTER_INTERFACE') + self.cluster_host_mtu = config.get( + 'cCLUSTER', 'CLUSTER_MTU') + self.cluster_host_vlan = '' + if config.has_option('cCLUSTER', 'CLUSTER_VLAN'): + cvalue = config.get('cCLUSTER', 'CLUSTER_VLAN') + if cvalue != 'NC': + self.cluster_host_vlan = cvalue + self.lag_cluster_host_interface = config.getboolean( + 'cCLUSTER', 'LAG_CLUSTER_INTERFACE') + if self.lag_cluster_host_interface: + self.lag_cluster_host_interface_member0 = config.get( + 'cCLUSTER', 'CLUSTER_BOND_MEMBER_0') + self.lag_cluster_host_interface_member1 = config.get( + 'cCLUSTER', 'CLUSTER_BOND_MEMBER_1') + self.lag_cluster_host_interface_policy = config.get( + 'cCLUSTER', 'CLUSTER_BOND_POLICY') + self.cluster_host_subnet = IPNetwork(config.get( + 'cCLUSTER', 'CLUSTER_SUBNET')) + self.cluster_host_interface_configured = True + # External OAM network configuration self.external_oam_interface_name = config.get( 'cEXT_OAM', 'EXTERNAL_OAM_INTERFACE_NAME') @@ -3090,6 +3454,29 @@ class ConfigAssistant(): print("Infrastructure end address: " + str(self.infrastructure_end_address)) + if self.kubernetes: + print("\nKubernetes Cluster Network Configuration") + print("----------------------------------------") + print("Cluster pod network subnet: " + + str(self.cluster_pod_subnet.cidr)) + print("Cluster service network subnet: " + + str(self.cluster_service_subnet.cidr)) + print("Cluster host interface name: " + + self.cluster_host_interface_name) + print("Cluster host interface: " + self.cluster_host_interface) + if self.infrastructure_vlan: + print("Cluster host vlan: " + self.cluster_host_vlan) + print("Cluster host interface MTU: " + self.cluster_host_mtu) + if self.lag_cluster_host_interface: + print("Cluster host ae member 0: " + + self.lag_cluster_host_interface_member0) + print("Cluster host ae member 1: " + + self.lag_cluster_host_interface_member1) + print("Cluster host ae policy : " + + self.lag_cluster_host_interface_policy) + print("Cluster host subnet: " + + str(self.cluster_host_subnet.cidr)) + print("\nExternal OAM Network Configuration") print("----------------------------------") print("External OAM interface name: " + @@ -3334,6 +3721,37 @@ class ConfigAssistant(): f.write("INFRASTRUCTURE_START_ADDRESS=NC\n") f.write("INFRASTRUCTURE_END_ADDRESS=NC\n") + # Cluster host network configuration + if self.kubernetes: + f.write("\n[cCLUSTER]") + f.write("\n# Cluster Host Network Configuration\n") + f.write("CLUSTER_INTERFACE_NAME=" + + self.cluster_host_interface_name + "\n") + f.write("CLUSTER_INTERFACE=" + + self.cluster_host_interface + "\n") + if self.cluster_host_vlan: + f.write("CLUSTER_VLAN=" + + self.cluster_host_vlan + "\n") + else: + f.write("CLUSTER_VLAN=NC\n") + f.write("CLUSTER_MTU=" + + self.cluster_host_mtu + "\n") + f.write("CLUSTER_SUBNET=" + + str(self.cluster_host_subnet.cidr) + "\n") + if self.lag_cluster_host_interface: + f.write("LAG_CLUSTER_INTERFACE=yes\n") + f.write("CLUSTER_BOND_MEMBER_0=" + + str(self.lag_cluster_host_interface_member0) + + "\n") + f.write("CLUSTER_BOND_MEMBER_1=" + + str(self.lag_cluster_host_interface_member1) + + "\n") + f.write("CLUSTER_BOND_POLICY=" + + str(self.lag_cluster_host_interface_policy) + + "\n") + else: + f.write("LAG_CLUSTER_INTERFACE=no\n") + # External OAM network configuration f.write("\n[cEXT_OAM]") f.write("\n# External OAM Network Configuration\n") @@ -4105,6 +4523,10 @@ class ConfigAssistant(): self._populate_infra_network(client) self._populate_oam_network(client) self._populate_multicast_network(client) + if self.kubernetes: + self._populate_cluster_host_network(client) + self._populate_cluster_pod_network(client) + self._populate_cluster_service_network(client) if self.subcloud_config(): self._populate_system_controller_network(client) @@ -4252,6 +4674,71 @@ class ConfigAssistant(): } client.sysinv.network.create(**values) + def _populate_cluster_host_network(self, client): + + # set default range if not specified as part of configuration + self.cluster_host_subnet_start_address = self.cluster_host_subnet[2] + self.cluster_host_subnet_end_address = self.cluster_host_subnet[-2] + + # create the address pool + values = { + 'name': 'cluster-host-subnet', + 'network': str(self.cluster_host_subnet.network), + 'prefix': self.cluster_host_subnet.prefixlen, + 'ranges': [(str(self.cluster_host_subnet_start_address), + str(self.cluster_host_subnet_end_address))], + } + pool = client.sysinv.address_pool.create(**values) + + # create the network for the pool + values = { + 'type': sysinv_constants.NETWORK_TYPE_CLUSTER_HOST, + 'name': sysinv_constants.NETWORK_TYPE_CLUSTER_HOST, + 'dynamic': self.dynamic_address_allocation, + 'pool_uuid': pool.uuid, + } + client.sysinv.network.create(**values) + + def _populate_cluster_pod_network(self, client): + # create the address pool + values = { + 'name': 'cluster-pod-subnet', + 'network': str(self.cluster_pod_subnet.network), + 'prefix': self.cluster_pod_subnet.prefixlen, + 'ranges': [(str(self.cluster_pod_subnet[1]), + str(self.cluster_pod_subnet[-2]))], + } + pool = client.sysinv.address_pool.create(**values) + + # create the network for the pool + values = { + 'type': sysinv_constants.NETWORK_TYPE_CLUSTER_POD, + 'name': sysinv_constants.NETWORK_TYPE_CLUSTER_POD, + 'dynamic': False, + 'pool_uuid': pool.uuid, + } + client.sysinv.network.create(**values) + + def _populate_cluster_service_network(self, client): + # create the address pool + values = { + 'name': 'cluster-service-subnet', + 'network': str(self.cluster_service_subnet.network), + 'prefix': self.cluster_service_subnet.prefixlen, + 'ranges': [(str(self.cluster_service_subnet[1]), + str(self.cluster_service_subnet[-2]))], + } + pool = client.sysinv.address_pool.create(**values) + + # create the network for the pool + values = { + 'type': sysinv_constants.NETWORK_TYPE_CLUSTER_SERVICE, + 'name': sysinv_constants.NETWORK_TYPE_CLUSTER_SERVICE, + 'dynamic': False, + 'pool_uuid': pool.uuid, + } + client.sysinv.network.create(**values) + def _populate_network_addresses(self, client, pool, network, addresses): for name, address in addresses.items(): values = { @@ -4326,6 +4813,8 @@ class ConfigAssistant(): self._populate_management_interface(client, controller) self._populate_infrastructure_interface(client, controller) self._populate_oam_interface(client, controller) + if self.kubernetes: + self._populate_cluster_host_interface(client, controller) def _update_interface_config(self, client, values): host_uuid = values.get('ihost_uuid') @@ -4401,6 +4890,9 @@ class ConfigAssistant(): if self.infrastructure_interface_configured: if ifname == self.infrastructure_interface: value = max(value, self.infrastructure_mtu) + if self.cluster_host_interface_configured: + if ifname == self.cluster_host_interface: + value = max(value, self.cluster_host_mtu) if self.external_oam_interface_configured: if ifname == self.external_oam_interface: value = max(value, self.external_oam_mtu) @@ -4557,6 +5049,75 @@ class ConfigAssistant(): } client.sysinv.iinterface.create(**values) + def _populate_cluster_host_interface(self, client, controller): + """Configure the cluster host interface(s)""" + network = self._get_network(client, + sysinv_constants.NETWORK_TYPE_CLUSTER_HOST) + + if (self.lag_cluster_host_interface and + self.cluster_host_interface_name != + self.management_interface_name): + members = [self.lag_cluster_host_interface_member0] + if self.lag_cluster_host_interface_member1: + members.append(self.lag_cluster_host_interface_member1) + + aemode = self._get_interface_aemode( + self.lag_cluster_host_interface_policy) + + txhashpolicy = self._get_interface_txhashpolicy( + self.lag_cluster_host_interface_policy) + + values = { + 'ihost_uuid': controller.uuid, + 'ifname': self.cluster_host_interface, + 'imtu': self._get_interface_mtu(self.cluster_host_interface), + 'iftype': sysinv_constants.INTERFACE_TYPE_AE, + 'aemode': aemode, + 'txhashpolicy': txhashpolicy, + 'ifclass': sysinv_constants.INTERFACE_CLASS_PLATFORM, + 'networks': [str(network.id)], + 'uses': members, + } + client.sysinv.iinterface.create(**values) + else: + # update MTU or network type of interface + values = { + 'ihost_uuid': controller.uuid, + 'ifname': self.cluster_host_interface, + } + values.update({ + 'imtu': self._get_interface_mtu(self.cluster_host_interface) + }) + if not self.cluster_host_vlan: + values.update({ + 'ifclass': sysinv_constants.INTERFACE_CLASS_PLATFORM, + 'networks_to_add': str(network.id), + }) + self._update_interface_config(client, values) + + if self.cluster_host_vlan: + if (self.cluster_host_interface_name != + self.management_interface_name): + values = { + 'ihost_uuid': controller.uuid, + 'ifname': self.cluster_host_interface_name, + 'imtu': self.cluster_host_mtu, + 'iftype': sysinv_constants.INTERFACE_TYPE_VLAN, + 'ifclass': sysinv_constants.INTERFACE_CLASS_PLATFORM, + 'networks': [str(network.id)], + 'uses': [self.cluster_host_interface], + 'vlan_id': self.cluster_host_vlan, + } + client.sysinv.iinterface.create(**values) + else: + values = { + 'ihost_uuid': controller.uuid, + 'ifname': self.cluster_host_interface_name, + 'ifclass': sysinv_constants.INTERFACE_CLASS_PLATFORM, + 'networks_to_add': str(network.id), + } + self._update_interface_config(client, values) + def _populate_oam_interface(self, client, controller): """Configure the OAM interface(s)""" diff --git a/controllerconfig/controllerconfig/controllerconfig/tests/files/system_config.cluster b/controllerconfig/controllerconfig/controllerconfig/tests/files/system_config.cluster new file mode 100755 index 0000000000..6117b7d4b4 --- /dev/null +++ b/controllerconfig/controllerconfig/controllerconfig/tests/files/system_config.cluster @@ -0,0 +1,59 @@ +[LOGICAL_INTERFACE_1] +LAG_INTERFACE=N +INTERFACE_MTU=1500 +INTERFACE_LINK_CAPACITY=1000 +INTERFACE_PORTS=eth1 + +[LOGICAL_INTERFACE_2] +LAG_INTERFACE=N +;LAG_MODE= +INTERFACE_MTU=1500 +;INTERFACE_LINK_CAPACITY= +INTERFACE_PORTS=eth0 + +[MGMT_NETWORK] +;VLAN=123 +CIDR=192.168.204.0/24 +MULTICAST_CIDR=239.1.1.0/28 +DYNAMIC_ALLOCATION=Y +LOGICAL_INTERFACE=LOGICAL_INTERFACE_1 + +[CLUSTER_NETWORK] +CIDR=192.168.206.0/24 +DYNAMIC_ALLOCATION=Y +IP_START_ADDRESS=192.168.206.2 +IP_END_ADDRESS=192.168.206.245 +LOGICAL_INTERFACE=LOGICAL_INTERFACE_1 + +;[INFRA_NETWORK] +;VLAN=124 +;IP_START_ADDRESS=192.168.205.102 +;IP_END_ADDRESS=192.168.205.199 +;DYNAMIC_ALLOCATION=Y +;CIDR=192.168.205.0/24 +;LOGICAL_INTERFACE=LOGICAL_INTERFACE_1 + +[OAM_NETWORK] +;VLAN= +;IP_START_ADDRESS=10.10.10.2 +;IP_END_ADDRESS=10.10.10.4 +IP_FLOATING_ADDRESS=10.10.10.20 +IP_UNIT_0_ADDRESS=10.10.10.30 +IP_UNIT_1_ADDRESS=10.10.10.40 +CIDR=10.10.10.0/24 +GATEWAY=10.10.10.1 +LOGICAL_INTERFACE=LOGICAL_INTERFACE_2 + +;[PXEBOOT_NETWORK] +;PXEBOOT_CIDR=192.168.203.0/24 + +;[BOARD_MANAGEMENT_NETWORK] +;VLAN=1 +;MTU=1496 +;SUBNET=192.168.203.0/24 + +[AUTHENTICATION] +ADMIN_PASSWORD=Li69nux* + +[VERSION] +RELEASE = TEST.SW.VERSION diff --git a/controllerconfig/controllerconfig/controllerconfig/tests/files/system_config.lag.vlan b/controllerconfig/controllerconfig/controllerconfig/tests/files/system_config.lag.vlan index 21b3c02f10..032c71ad7b 100755 --- a/controllerconfig/controllerconfig/controllerconfig/tests/files/system_config.lag.vlan +++ b/controllerconfig/controllerconfig/controllerconfig/tests/files/system_config.lag.vlan @@ -32,6 +32,13 @@ IP_END_ADDRESS=192.168.205.199 CIDR=192.168.205.0/24 LOGICAL_INTERFACE=LOGICAL_INTERFACE_1 +[CLUSTER_NETWORK] +VLAN=126 +IP_START_ADDRESS=192.168.206.102 +IP_END_ADDRESS=192.168.206.199 +CIDR=192.168.206.0/24 +LOGICAL_INTERFACE=LOGICAL_INTERFACE_1 + [OAM_NETWORK] VLAN=125 IP_START_ADDRESS=10.10.10.2 diff --git a/controllerconfig/controllerconfig/controllerconfig/tests/test_system_config.py b/controllerconfig/controllerconfig/controllerconfig/tests/test_system_config.py index c662607213..eda2692230 100644 --- a/controllerconfig/controllerconfig/controllerconfig/tests/test_system_config.py +++ b/controllerconfig/controllerconfig/controllerconfig/tests/test_system_config.py @@ -387,6 +387,46 @@ def test_system_config_validation(): with pytest.raises(exceptions.ConfigFail): validate(system_config, DEFAULT_CONFIG, None, False) + # Test overlap of CLUSTER_NETWORK CIDR + system_config = cr.parse_system_config(lag_vlan_systemfile) + system_config.set('CLUSTER_NETWORK', 'CIDR', '192.168.203.0/26') + with pytest.raises(exceptions.ConfigFail): + cr.create_cgcs_config_file(None, system_config, None, None, None, 0, + validate_only=True) + with pytest.raises(exceptions.ConfigFail): + validate(system_config, DEFAULT_CONFIG, None, False) + + system_config.set('CLUSTER_NETWORK', 'CIDR', '192.168.204.0/26') + with pytest.raises(exceptions.ConfigFail): + cr.create_cgcs_config_file(None, system_config, None, None, None, 0, + validate_only=True) + with pytest.raises(exceptions.ConfigFail): + validate(system_config, DEFAULT_CONFIG, None, False) + + # Test invalid CLUSTER_NETWORK LAG_MODE + system_config = cr.parse_system_config(lag_vlan_systemfile) + system_config.add_section('LOGICAL_INTERFACE_2') + system_config.set('LOGICAL_INTERFACE_2', 'LAG_INTERFACE', 'Y') + system_config.set('LOGICAL_INTERFACE_2', 'LAG_MODE', '3') + system_config.set('LOGICAL_INTERFACE_2', 'INTERFACE_MTU', '1500') + system_config.set('LOGICAL_INTERFACE_2', 'INTERFACE_PORTS', 'eth3,eth4') + system_config.set('CLUSTER_NETWORK', 'LOGICAL_INTERFACE', + 'LOGICAL_INTERFACE_2') + with pytest.raises(exceptions.ConfigFail): + cr.create_cgcs_config_file(None, system_config, None, None, None, 0, + validate_only=True) + with pytest.raises(exceptions.ConfigFail): + validate(system_config, DEFAULT_CONFIG, None, False) + + # Test CLUSTER_NETWORK VLAN overlap + system_config = cr.parse_system_config(lag_vlan_systemfile) + system_config.set('CLUSTER_NETWORK', 'VLAN', '123') + with pytest.raises(exceptions.ConfigFail): + cr.create_cgcs_config_file(None, system_config, None, None, None, 0, + validate_only=True) + with pytest.raises(exceptions.ConfigFail): + validate(system_config, DEFAULT_CONFIG, None, False) + # Test overlap of OAM_NETWORK CIDR system_config = cr.parse_system_config(lag_vlan_systemfile) system_config.set('OAM_NETWORK', 'CIDR', '192.168.203.0/26') @@ -564,3 +604,46 @@ def test_pxeboot_range(): system_config.set('PXEBOOT_NETWORK', 'IP_END_ADDRESS', '128.123.122.34') with pytest.raises(exceptions.ConfigFail): validate(system_config, DEFAULT_CONFIG, None, False) + + +def test_cluster_network(): + """ Test import of system_config file for cluster network address """ + + # Create the path to the system_config file + systemfile = os.path.join( + os.getcwd(), "controllerconfig/tests/files/", "system_config.cluster") + + # Test import and generation of answer file + _test_system_config(systemfile) + + # Test CLUSTER_NETWORK start address specified without end address + system_config = cr.parse_system_config(systemfile) + system_config.set('CLUSTER_NETWORK', 'IP_START_ADDRESS', '192.168.204.2') + with pytest.raises(exceptions.ConfigFail): + cr.create_cgcs_config_file(None, system_config, None, None, None, 0, + validate_only=True) + with pytest.raises(exceptions.ConfigFail): + validate(system_config, DEFAULT_CONFIG, None, False) + + # Test CLUSTER_NETWORK end address specified without start address + system_config = cr.parse_system_config(systemfile) + system_config.set('CLUSTER_NETWORK', 'IP_END_ADDRESS', '192.168.204.200') + with pytest.raises(exceptions.ConfigFail): + cr.create_cgcs_config_file(None, system_config, None, None, None, 0, + validate_only=True) + with pytest.raises(exceptions.ConfigFail): + validate(system_config, DEFAULT_CONFIG, None, False) + + # Test detection of overspecification of CLUSTER network addresses + system_config = cr.parse_system_config(systemfile) + system_config.set('CLUSTER_NETWORK', 'IP_FLOATING_ADDRESS', + '192.168.206.103') + system_config.set('CLUSTER_NETWORK', 'IP_IP_UNIT_0_ADDRESS', + '192.168.206.106') + system_config.set('CLUSTER_NETWORK', 'IP_IP_UNIT_1_ADDRESS', + '192.168.206.109') + with pytest.raises(exceptions.ConfigFail): + cr.create_cgcs_config_file(None, system_config, None, None, None, 0, + validate_only=True) + with pytest.raises(exceptions.ConfigFail): + validate(system_config, DEFAULT_CONFIG, None, False) diff --git a/puppet-manifests/src/modules/platform/manifests/config.pp b/puppet-manifests/src/modules/platform/manifests/config.pp index eec5e6b969..ed82c8bc1c 100644 --- a/puppet-manifests/src/modules/platform/manifests/config.pp +++ b/puppet-manifests/src/modules/platform/manifests/config.pp @@ -34,10 +34,17 @@ class platform::config::file { include ::platform::network::mgmt::params include ::platform::network::infra::params include ::platform::network::oam::params + include ::platform::network::cluster_host::params + include ::platform::kubernetes::params + $kubernetes_enabled = $::platform::kubernetes::params::enabled # dependent template variables $management_interface = $::platform::network::mgmt::params::interface_name - $infrastructure_interface = $::platform::network::infra::params::interface_name + if $kubernetes_enabled { + $infrastructure_interface = $::platform::network::cluster_host::params::interface_name + } else { + $infrastructure_interface = $::platform::network::infra::params::interface_name + } $oam_interface = $::platform::network::oam::params::interface_name $platform_conf = '/etc/platform/platform.conf' diff --git a/puppet-manifests/src/modules/platform/manifests/dns.pp b/puppet-manifests/src/modules/platform/manifests/dns.pp index 17d3fd48c2..786c89a7c8 100644 --- a/puppet-manifests/src/modules/platform/manifests/dns.pp +++ b/puppet-manifests/src/modules/platform/manifests/dns.pp @@ -63,6 +63,16 @@ class platform::dns::dnsmasq { } } + include ::platform::kubernetes::params + $kubernetes_enabled = $::platform::kubernetes::params::enabled + if $kubernetes_enabled { + $service_domain = $::platform::kubernetes::params::service_domain + $dns_service_ip = $::platform::kubernetes::params::dns_service_ip + } else { + $service_domain = undef + $dns_service_ip = undef + } + file { '/etc/dnsmasq.conf': ensure => 'present', replace => true, diff --git a/puppet-manifests/src/modules/platform/manifests/kubernetes.pp b/puppet-manifests/src/modules/platform/manifests/kubernetes.pp index cc70639c93..d132dd8766 100644 --- a/puppet-manifests/src/modules/platform/manifests/kubernetes.pp +++ b/puppet-manifests/src/modules/platform/manifests/kubernetes.pp @@ -1,8 +1,11 @@ class platform::kubernetes::params ( $enabled = false, $pod_network_cidr = undef, + $service_network_cidr = undef, $apiserver_advertise_address = undef, $etcd_endpoint = undef, + $service_domain = undef, + $dns_service_ip = undef, $ca_crt = undef, $ca_key = undef, $sa_key = undef, @@ -45,18 +48,6 @@ class platform::kubernetes::kubeadm { command => 'sysctl --system', } - # TODO: Update /etc/resolv.conf.k8s to be controlled by sysinv, as is done - # for /etc/resolv.conf. Is should contain all the user-specified DNS - # servers, but not the coredns IP. - # Create custom resolv.conf file for kubelet - -> file { '/etc/resolv.conf.k8s': - ensure => file, - content => 'nameserver 8.8.8.8', - owner => 'root', - group => 'root', - mode => '0644', - } - # Start kubelet. -> service { 'kubelet': ensure => 'running', diff --git a/puppet-manifests/src/modules/platform/manifests/network.pp b/puppet-manifests/src/modules/platform/manifests/network.pp index 80b5a921a8..6fa346b6bc 100644 --- a/puppet-manifests/src/modules/platform/manifests/network.pp +++ b/puppet-manifests/src/modules/platform/manifests/network.pp @@ -81,6 +81,24 @@ class platform::network::oam::params( $mtu = 1500, ) { } +class platform::network::cluster_host::params( + # shared parametes with base class - required for auto hiera parameter lookup + $interface_name = undef, + $interface_address = undef, + $subnet_version = undef, + $subnet_network = undef, + $subnet_network_url = undef, + $subnet_prefixlen = undef, + $subnet_netmask = undef, + $subnet_start = undef, + $subnet_end = undef, + $gateway_address = undef, + $controller_address = undef, # controller floating + $controller_address_url = undef, # controller floating url address + $controller0_address = undef, # controller unit0 + $controller1_address = undef, # controller unit1 + $mtu = 1500, +) { } define network_address ( $address, @@ -148,6 +166,7 @@ class platform::network ( include ::platform::params include ::platform::network::mgmt::params include ::platform::network::infra::params + include ::platform::network::cluster_host::params include ::platform::network::apply diff --git a/puppet-manifests/src/modules/platform/manifests/sm.pp b/puppet-manifests/src/modules/platform/manifests/sm.pp old mode 100755 new mode 100644 index 9302340bac..90242d6e72 --- a/puppet-manifests/src/modules/platform/manifests/sm.pp +++ b/puppet-manifests/src/modules/platform/manifests/sm.pp @@ -15,6 +15,9 @@ class platform::sm $system_mode = $::platform::params::system_mode $system_type = $::platform::params::system_type + include ::platform::kubernetes::params + $kubernetes_enabled = $::platform::kubernetes::params::enabled + include ::platform::network::pxeboot::params if $::platform::network::pxeboot::params::interface_name { $pxeboot_ip_interface = $::platform::network::pxeboot::params::interface_name @@ -30,8 +33,13 @@ class platform::sm $mgmt_ip_param_ip = $::platform::network::mgmt::params::controller_address $mgmt_ip_param_mask = $::platform::network::mgmt::params::subnet_prefixlen - include ::platform::network::infra::params - $infra_ip_interface = $::platform::network::infra::params::interface_name + if $kubernetes_enabled { + # Repurposing the infra interface for cluster-host interface + include ::platform::network::cluster_host::params + $infra_ip_interface = $::platform::network::cluster_host::params::interface_name + } else { + $infra_ip_interface = $::platform::network::infra::params::interface_name + } include ::platform::network::oam::params $oam_ip_interface = $::platform::network::oam::params::interface_name @@ -113,9 +121,6 @@ class platform::sm $rabbitmq_server = '/usr/lib/rabbitmq/bin/rabbitmq-server' $rabbitmqctl = '/usr/lib/rabbitmq/bin/rabbitmqctl' - include ::platform::kubernetes::params - $kubernetes_enabled = $::platform::kubernetes::params::enabled - include ::platform::mtce::params $sm_client_port = $::platform::mtce::params::sm_client_port $sm_server_port = $::platform::mtce::params::sm_server_port @@ -129,14 +134,14 @@ class platform::sm $platform_nfs_ip_network_url = $::platform::network::mgmt::params::subnet_network_url # CGCS NFS network is over the infrastructure network if configured - if $infra_ip_interface { - $cgcs_nfs_ip_interface = $::platform::network::infra::params::interface_name - $cgcs_nfs_ip_param_ip = $::platform::network::infra::params::cgcs_nfs_address - $cgcs_nfs_ip_network_url = $::platform::network::infra::params::subnet_network_url - $cgcs_nfs_ip_param_mask = $::platform::network::infra::params::subnet_prefixlen + if $infra_ip_interface and $kubernetes_enabled != true { + $cgcs_nfs_ip_interface = $::platform::network::infra::params::interface_name + $cgcs_nfs_ip_param_ip = $::platform::network::infra::params::cgcs_nfs_address + $cgcs_nfs_ip_network_url = $::platform::network::infra::params::subnet_network_url + $cgcs_nfs_ip_param_mask = $::platform::network::infra::params::subnet_prefixlen - $cinder_ip_interface = $::platform::network::infra::params::interface_name - $cinder_ip_param_mask = $::platform::network::infra::params::subnet_prefixlen + $cinder_ip_interface = $::platform::network::infra::params::interface_name + $cinder_ip_param_mask = $::platform::network::infra::params::subnet_prefixlen } else { $cgcs_nfs_ip_interface = $::platform::network::mgmt::params::interface_name $cgcs_nfs_ip_param_ip = $::platform::network::mgmt::params::cgcs_nfs_address @@ -247,6 +252,10 @@ class platform::sm $hostunit = '0' $management_my_unit_ip = $::platform::network::mgmt::params::controller0_address $oam_my_unit_ip = $::platform::network::oam::params::controller_address + if $kubernetes_enabled { + # Repurposing the infra interface for cluster-host interface + $infra_my_unit_ip = $::platform::network::cluster_host::params::controller_address + } } else { case $::hostname { $controller_0_hostname: { @@ -255,8 +264,14 @@ class platform::sm $management_peer_unit_ip = $::platform::network::mgmt::params::controller1_address $oam_my_unit_ip = $::platform::network::oam::params::controller0_address $oam_peer_unit_ip = $::platform::network::oam::params::controller1_address - $infra_my_unit_ip = $::platform::network::infra::params::controller0_address - $infra_peer_unit_ip = $::platform::network::infra::params::controller1_address + if $kubernetes_enabled { + # Repurposing the infra interface for cluster-host interface + $infra_my_unit_ip = $::platform::network::cluster_host::params::controller0_address + $infra_peer_unit_ip = $::platform::network::cluster_host::params::controller1_address + } else { + $infra_my_unit_ip = $::platform::network::infra::params::controller0_address + $infra_peer_unit_ip = $::platform::network::infra::params::controller1_address + } } $controller_1_hostname: { $hostunit = '1' @@ -264,8 +279,14 @@ class platform::sm $management_peer_unit_ip = $::platform::network::mgmt::params::controller0_address $oam_my_unit_ip = $::platform::network::oam::params::controller1_address $oam_peer_unit_ip = $::platform::network::oam::params::controller0_address - $infra_my_unit_ip = $::platform::network::infra::params::controller1_address - $infra_peer_unit_ip = $::platform::network::infra::params::controller0_address + if $kubernetes_enabled { + # Repurposing the infra interface for cluster-host interface + $infra_my_unit_ip = $::platform::network::cluster_host::params::controller1_address + $infra_peer_unit_ip = $::platform::network::cluster_host::params::controller0_address + } else { + $infra_my_unit_ip = $::platform::network::infra::params::controller1_address + $infra_peer_unit_ip = $::platform::network::infra::params::controller0_address + } } default: { $hostunit = '2' @@ -325,13 +346,25 @@ class platform::sm exec { 'Configure Management Interface': command => "sm-configure interface controller management-interface ${mgmt_ip_multicast} ${management_my_unit_ip} 2222 2223 \"\" 2222 2223", } - } else { - exec { 'Configure OAM Interface': - command => "sm-configure interface controller oam-interface \"\" ${oam_my_unit_ip} 2222 2223 ${oam_peer_unit_ip} 2222 2223", + + if $kubernetes_enabled { + exec { 'Configure Cluster Host Interface': + command => "sm-configure interface controller infrastructure-interface \"\" ${infra_my_unit_ip} 2222 2223 \"\" 2222 2223", } + } + + } else { + exec { 'Configure OAM Interface': + command => "sm-configure interface controller oam-interface \"\" ${oam_my_unit_ip} 2222 2223 ${oam_peer_unit_ip} 2222 2223", + } exec { 'Configure Management Interface': command => "sm-configure interface controller management-interface ${mgmt_ip_multicast} ${management_my_unit_ip} 2222 2223 ${management_peer_unit_ip} 2222 2223", } + if $kubernetes_enabled or $infra_ip_interface { + exec { 'Configure Infrastructure Interface': + command => "sm-configure interface controller infrastructure-interface ${infra_ip_multicast} ${infra_my_unit_ip} 2222 2223 ${infra_peer_unit_ip} 2222 2223", + } + } } exec { 'Configure OAM IP': @@ -1147,12 +1180,6 @@ class platform::sm command => "sm-configure service_instance open-ldap open-ldap \"\"", } - if $infra_ip_interface { - exec { 'Configure Infrastructure Interface': - command => "sm-configure interface controller infrastructure-interface ${infra_ip_multicast} ${infra_my_unit_ip} 2222 2223 ${infra_peer_unit_ip} 2222 2223", - } - } - if $system_mode == 'duplex-direct' or $system_mode == 'duplex' { exec { 'Configure System Mode': command => "sm-configure system --cpe_mode ${system_mode}", diff --git a/puppet-manifests/src/modules/platform/templates/dnsmasq.conf.erb b/puppet-manifests/src/modules/platform/templates/dnsmasq.conf.erb index c2e2329811..7fd5f7c80c 100644 --- a/puppet-manifests/src/modules/platform/templates/dnsmasq.conf.erb +++ b/puppet-manifests/src/modules/platform/templates/dnsmasq.conf.erb @@ -131,3 +131,8 @@ dhcp-leasefile=<%= @config_path %>/dnsmasq.leases addn-hosts=<%= @config_path %>/dnsmasq.addn_hosts # File for distributed cloud subcloud ip translation addn-hosts=<%= @config_path %>/dnsmasq.addn_hosts_dc + +# Set server to Kubernetes coredns cluster IP +<%- if @dns_service_ip != nil -%> +server=/<%= @service_domain %>/<%= @dns_service_ip %> +<%- end -%> diff --git a/puppet-manifests/src/modules/platform/templates/kubeadm.yaml.erb b/puppet-manifests/src/modules/platform/templates/kubeadm.yaml.erb index 709156ade6..c64b03471b 100644 --- a/puppet-manifests/src/modules/platform/templates/kubeadm.yaml.erb +++ b/puppet-manifests/src/modules/platform/templates/kubeadm.yaml.erb @@ -14,7 +14,9 @@ apiServerCertSANs: - "<%= @apiserver_advertise_address %>" - "127.0.0.1" networking: + dnsDomain: <%= @service_domain %> podSubnet: <%= @pod_network_cidr %> + serviceSubnet: <%= @service_network_cidr %> controllerManagerExtraArgs: node-monitor-period: "2s" node-monitor-grace-period: "20s" @@ -26,4 +28,3 @@ nodeStatusUpdateFrequency: "4s" failSwapOn: false featureGates: HugePages: false -resolvConf: "/etc/resolv.conf.k8s" diff --git a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/address.py b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/address.py index c3924023fa..60c6563340 100644 --- a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/address.py +++ b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/address.py @@ -46,6 +46,7 @@ LOG = log.getLogger(__name__) ALLOWED_NETWORK_TYPES = [constants.NETWORK_TYPE_MGMT, constants.NETWORK_TYPE_INFRA, constants.NETWORK_TYPE_OAM, + constants.NETWORK_TYPE_CLUSTER_HOST, constants.NETWORK_TYPE_DATA] diff --git a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/address_pool.py b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/address_pool.py index 066b40c352..08416e1b6c 100644 --- a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/address_pool.py +++ b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/address_pool.py @@ -466,7 +466,6 @@ class AddressPoolController(rest.RestController): addrpool_dict = addrpool.as_dict() self._set_defaults(addrpool_dict) self._sort_ranges(addrpool_dict) - # Check for semantic conflicts self._check_name_conflict(addrpool_dict) self._check_valid_ranges(addrpool_dict) diff --git a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/interface.py b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/interface.py index 54e2c0dbea..a1c79789bf 100644 --- a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/interface.py +++ b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/interface.py @@ -64,6 +64,7 @@ VALID_NETWORK_TYPES = [constants.NETWORK_TYPE_NONE, constants.NETWORK_TYPE_OAM, constants.NETWORK_TYPE_MGMT, constants.NETWORK_TYPE_INFRA, + constants.NETWORK_TYPE_CLUSTER_HOST, constants.NETWORK_TYPE_DATA, constants.NETWORK_TYPE_PCI_PASSTHROUGH, constants.NETWORK_TYPE_PCI_SRIOV] @@ -698,6 +699,8 @@ class InterfaceController(rest.RestController): _update_host_mgmt_address(ihost, interface) if constants.NETWORK_TYPE_INFRA in networktypelist: _update_host_infra_address(ihost, interface) + if constants.NETWORK_TYPE_CLUSTER_HOST in networktypelist: + _update_host_cluster_address(ihost, interface) if ihost['personality'] == constants.CONTROLLER: if constants.NETWORK_TYPE_OAM in networktypelist: _update_host_oam_address(ihost, interface) @@ -1385,10 +1388,8 @@ def _check_interface_data(op, interface, ihost, existing_interface): # Make sure interface type is valid supported_type = [constants.INTERFACE_TYPE_AE, constants.INTERFACE_TYPE_VLAN, - constants.INTERFACE_TYPE_ETHERNET] - # only allows add operation for the virtual interface - if op == 'add': - supported_type.append(constants.INTERFACE_TYPE_VIRTUAL) + constants.INTERFACE_TYPE_ETHERNET, + constants.INTERFACE_TYPE_VIRTUAL] if not iftype or iftype not in supported_type: msg = (_("Device interface type must be one of " "{}").format(', '.join(supported_type))) @@ -1706,7 +1707,8 @@ def _delete_addressing(interface, family, existing_interface): interface['id'] ) elif ((orig_networktype != constants.NETWORK_TYPE_MGMT) and - (orig_networktype != constants.NETWORK_TYPE_INFRA)): + (orig_networktype != constants.NETWORK_TYPE_CLUSTER_HOST) and + (orig_networktype != constants.NETWORK_TYPE_INFRA)): pecan.request.dbapi.addresses_destroy_by_interface( interface_id, family) pecan.request.dbapi.address_modes_destroy_by_interface( @@ -1871,6 +1873,14 @@ def _update_host_pxeboot_address(host, interface): pecan.request.dbapi.address_update(address.uuid, updates) +def _update_host_cluster_address(host, interface): + address_name = cutils.format_address_name( + host.hostname, constants.NETWORK_TYPE_CLUSTER_HOST) + address = pecan.request.dbapi.address_get_by_name(address_name) + updates = {'interface_id': interface['id']} + pecan.request.dbapi.address_update(address.uuid, updates) + + def _clean_providernetworks(providernetworks): pn = [','.join(p['name']) for p in providernetworks] return pn @@ -2329,11 +2339,15 @@ def _create(interface, from_profile=False): _update_host_mgmt_address(ihost, new_interface.as_dict()) elif network.type == constants.NETWORK_TYPE_INFRA: _update_host_infra_address(ihost, new_interface.as_dict()) + elif network.type == constants.NETWORK_TYPE_CLUSTER_HOST: + _update_host_cluster_address(ihost, + new_interface.as_dict()) if ihost['personality'] == constants.CONTROLLER: if network.type == constants.NETWORK_TYPE_OAM: _update_host_oam_address(ihost, new_interface.as_dict()) elif network.type == constants.NETWORK_TYPE_PXEBOOT: _update_host_pxeboot_address(ihost, new_interface.as_dict()) + except Exception as e: LOG.exception( "Failed to add static infrastructure interface address: " diff --git a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/network.py b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/network.py index fa14c41f72..662c6aad3e 100644 --- a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/network.py +++ b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/network.py @@ -48,7 +48,11 @@ ALLOWED_NETWORK_TYPES = [constants.NETWORK_TYPE_MGMT, constants.NETWORK_TYPE_INFRA, constants.NETWORK_TYPE_OAM, constants.NETWORK_TYPE_MULTICAST, - constants.NETWORK_TYPE_SYSTEM_CONTROLLER] + constants.NETWORK_TYPE_SYSTEM_CONTROLLER, + constants.NETWORK_TYPE_CLUSTER_HOST, + constants.NETWORK_TYPE_CLUSTER_POD, + constants.NETWORK_TYPE_CLUSTER_SERVICE, + ] class Network(base.APIBase): @@ -174,12 +178,15 @@ class NetworkController(rest.RestController): elif network['type'] == constants.NETWORK_TYPE_INFRA: addresses = self._create_infra_network_address() self._remove_mgmt_cinder_address() + elif network['type'] == constants.NETWORK_TYPE_CLUSTER_HOST: + addresses = self._create_cluster_host_network_address() elif network['type'] == constants.NETWORK_TYPE_OAM: addresses = self._create_oam_network_address(pool) elif network['type'] == constants.NETWORK_TYPE_MULTICAST: addresses = self._create_multicast_network_address() elif network['type'] == constants.NETWORK_TYPE_SYSTEM_CONTROLLER: addresses = self._create_system_controller_network_address(pool) + else: return self._populate_network_addresses(pool, network, addresses) @@ -219,6 +226,13 @@ class NetworkController(rest.RestController): addresses[constants.CONTROLLER_CINDER] = None return addresses + def _create_cluster_host_network_address(self): + addresses = collections.OrderedDict() + addresses[constants.CONTROLLER_HOSTNAME] = None + addresses[constants.CONTROLLER_0_HOSTNAME] = None + addresses[constants.CONTROLLER_1_HOSTNAME] = None + return addresses + def _remove_mgmt_cinder_address(self): # Remove old cinder's IP from the management network try: diff --git a/sysinv/sysinv/sysinv/sysinv/common/constants.py b/sysinv/sysinv/sysinv/sysinv/common/constants.py index 7c160d23bf..fe6b0cbf5f 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/constants.py +++ b/sysinv/sysinv/sysinv/sysinv/common/constants.py @@ -583,6 +583,9 @@ NETWORK_TYPE_BM = 'bm' NETWORK_TYPE_MULTICAST = 'multicast' NETWORK_TYPE_DATA = 'data' NETWORK_TYPE_SYSTEM_CONTROLLER = 'system-controller' +NETWORK_TYPE_CLUSTER_HOST = 'cluster-host' +NETWORK_TYPE_CLUSTER_POD = 'cluster-pod' +NETWORK_TYPE_CLUSTER_SERVICE = 'cluster-service' NETWORK_TYPE_PCI_PASSTHROUGH = 'pci-passthrough' NETWORK_TYPE_PCI_SRIOV = 'pci-sriov' @@ -591,7 +594,8 @@ NETWORK_TYPE_PXEBOOT = 'pxeboot' PLATFORM_NETWORK_TYPES = [NETWORK_TYPE_PXEBOOT, NETWORK_TYPE_MGMT, NETWORK_TYPE_INFRA, - NETWORK_TYPE_OAM] + NETWORK_TYPE_OAM, + NETWORK_TYPE_CLUSTER_HOST] PCI_NETWORK_TYPES = [NETWORK_TYPE_PCI_PASSTHROUGH, NETWORK_TYPE_PCI_SRIOV] @@ -1478,3 +1482,7 @@ APP_TARFILE_NAME_PLACEHOLDER = 'tarfile-placeholder' CONTROL_PLANE_LABEL = 'openstack-control-plane=enabled' COMPUTE_NODE_LABEL = 'openstack-compute-node=enabled' OPENVSWITCH_LABEL = 'openvswitch=enabled' + +# Default DNS service domain +DEFAULT_DNS_SERVICE_DOMAIN = 'cluster.local' +DEFAULT_DNS_SERVICE_IP = '10.96.0.10' diff --git a/sysinv/sysinv/sysinv/sysinv/helm/nova.py b/sysinv/sysinv/sysinv/sysinv/helm/nova.py index f13e4839f0..08e5f9a8f4 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/nova.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/nova.py @@ -324,33 +324,35 @@ class NovaHelm(openstack.OpenstackBaseHelm): def _update_host_addresses(self, host, default_config, vnc_config, libvirt_config): interfaces = self.dbapi.iinterface_get_by_ihost(host.id) addresses = self.dbapi.addresses_get_by_host(host.id) - - primary_infra = None - primary_mgmt = None + cluster_host_network = self.dbapi.network_get_by_type( + constants.NETWORK_TYPE_CLUSTER_HOST) + cluster_host_iface = None for iface in interfaces: - if constants.NETWORK_TYPE_INFRA == utils.get_primary_network_type(iface): - primary_infra = iface - if constants.NETWORK_TYPE_MGMT == utils.get_primary_network_type(iface): - primary_mgmt = iface + interface_network = {'interface_id': iface.id, + 'network_id': cluster_host_network.id} + try: + self.dbapi.interface_network_query(interface_network) + cluster_host_iface = iface + except exception.InterfaceNetworkNotFoundByHostInterfaceNetwork: + pass - migration_iface = primary_infra or primary_mgmt - if migration_iface is None: + if cluster_host_iface is None: return + cluster_host_ip = None + ip_family = None for addr in addresses: - if addr.interface_uuid == migration_iface.uuid: - migration_ip = addr.address + if addr.interface_uuid == cluster_host_iface.uuid: + cluster_host_ip = addr.address ip_family = addr.family - if addr.interface_uuid == primary_mgmt.uuid: - mgmt_ip = addr.address - default_config.update({'my_ip': migration_ip}) + default_config.update({'my_ip': cluster_host_ip}) if ip_family == 4: vnc_config.update({'vncserver_listen': '0.0.0.0'}) elif ip_family == 6: vnc_config.update({'vncserver_listen': '::0'}) - libvirt_config.update({'live_migration_inbound_addr': migration_ip}) - vnc_config.update({'vncserver_proxyclient_address': mgmt_ip}) + libvirt_config.update({'live_migration_inbound_addr': cluster_host_ip}) + vnc_config.update({'vncserver_proxyclient_address': cluster_host_ip}) def _update_host_memory(self, host, default_config): vswitch_2M_pages = [] diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/base.py b/sysinv/sysinv/sysinv/sysinv/puppet/base.py index bfd32e7d50..b26d40e3dc 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/base.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/base.py @@ -189,6 +189,17 @@ class BasePuppet(object): constants.CONTROLLER_HOSTNAME, constants.NETWORK_TYPE_OAM) return address.address + def _get_cluster_host_address(self): + address = self._get_address_by_name( + constants.CONTROLLER_HOSTNAME, constants.NETWORK_TYPE_CLUSTER_HOST) + return address.address + + def _get_cluster_pod_subnet(self): + address = self._get_address_by_name( + constants.CONTROLLER_HOSTNAME, constants.NETWORK_TYPE_CLUSTER_POD) + subnet = address.address + '/' + address.prefix + return subnet + def _get_host_cpu_list(self, host, function=None, threads=False): """ Retreive a list of CPUs for the host, filtered by function and thread diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/interface.py b/sysinv/sysinv/sysinv/sysinv/puppet/interface.py index 6b305f4395..303e3122f6 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/interface.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/interface.py @@ -25,6 +25,7 @@ LOG = log.getLogger(__name__) PLATFORM_NETWORK_TYPES = [constants.NETWORK_TYPE_PXEBOOT, constants.NETWORK_TYPE_MGMT, constants.NETWORK_TYPE_INFRA, + constants.NETWORK_TYPE_CLUSTER_HOST, constants.NETWORK_TYPE_OAM] DATA_NETWORK_TYPES = [constants.NETWORK_TYPE_DATA] @@ -118,6 +119,7 @@ class InterfacePuppet(base.BasePuppet): 'personality': host.personality, 'subfunctions': host.subfunctions, 'system_uuid': host.isystem_uuid, + 'system_mode': self._get_system().system_mode, 'ports': self._get_port_interface_id_index(host), 'interfaces': self._get_interface_name_index(host), 'devices': self._get_port_pciaddr_index(host), @@ -260,6 +262,19 @@ class InterfacePuppet(base.BasePuppet): constants.NETWORK_TYPE_OAM: oam_floating_ip }) + try: + cluster_address = self._get_address_by_name( + constants.CONTROLLER_HOSTNAME, + constants.NETWORK_TYPE_CLUSTER_HOST) + if cluster_address: + cluster_floating_ip = (str(cluster_address.address) + '/' + + str(cluster_address.prefix)) + floating_ips.update({ + constants.NETWORK_TYPE_CLUSTER_HOST: cluster_floating_ip + }) + except exception.AddressNotFoundByName: + pass + return floating_ips def _get_provider_networks(self, host): @@ -595,6 +610,8 @@ def get_interface_address_method(context, iface): # All other interface types that exist on a controller are setup # statically since the controller themselves run the DHCP server. return 'static' + elif iface.networktype == constants.NETWORK_TYPE_CLUSTER_HOST: + return 'static' elif iface.networktype == constants.NETWORK_TYPE_PXEBOOT: # All pxeboot interfaces that exist on non-controller nodes are set # to manual as they are not needed/used once the install is done. @@ -872,12 +889,11 @@ def get_common_network_config(context, iface, config, network_id=None): method = get_interface_address_method(context, iface) if method == 'static': address = get_interface_primary_address(context, iface, network_id) - if address is None: - LOG.error("Interface %s has no primary address" % iface['ifname']) - assert address is not None - - config['ipaddress'] = address['address'] - config['netmask'] = address['netmask'] + if address: + config['ipaddress'] = address['address'] + config['netmask'] = address['netmask'] + else: + LOG.info("Interface %s has no primary address" % iface['ifname']) if network_id is None and len(iface.networks) > 0: networktype = find_networktype_by_network_id( @@ -931,6 +947,9 @@ def generate_network_config(context, config, iface): net_config = get_interface_network_config(context, iface, int(net_id)) ifname = net_config['ifname'] + ':' + net_id + if context['system_mode'] == constants.SYSTEM_MODE_SIMPLEX: + options = {'SCOPE': 'scope host'} + net_config['options'].update(options) config[NETWORK_CONFIG_RESOURCE].update({ ifname: format_network_config(net_config) }) @@ -1049,7 +1068,7 @@ def generate_address_configs(context, config): networktype: address_config }) elif networktype == constants.NETWORK_TYPE_PXEBOOT: - # Fallback PXE boot address against mananagement interface + # Fallback PXE boot address against management interface iface = find_interface_by_type(context, constants.NETWORK_TYPE_MGMT) if iface: @@ -1057,6 +1076,15 @@ def generate_address_configs(context, config): config[ADDRESS_CONFIG_RESOURCE].update({ networktype: address_config }) + elif networktype == constants.NETWORK_TYPE_CLUSTER_HOST: + # Fallback cluster host address against management interface + iface = find_interface_by_type(context, + constants.NETWORK_TYPE_CLUSTER_HOST) + if iface: + address_config = get_address_config(context, iface, address) + config[ADDRESS_CONFIG_RESOURCE].update({ + networktype: address_config + }) def build_mlx4_num_vfs_options(context): diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/kubernetes.py b/sysinv/sysinv/sysinv/sysinv/puppet/kubernetes.py index a1c3fe76d0..aeb48f558e 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/kubernetes.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/kubernetes.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: Apache-2.0 # +from __future__ import absolute_import import os import subprocess @@ -25,15 +26,18 @@ class KubernetesPuppet(base.BasePuppet): if self._kubernetes_enabled(): config.update( {'platform::kubernetes::params::enabled': True, - # Hardcoding the pod network cidr to a private network that - # does not overlap with our default management network. Will - # eventually be configurable. 'platform::kubernetes::params::pod_network_cidr': - '172.16.0.0/16', + self._get_pod_network_cidr(), + 'platform::kubernetes::params::service_network_cidr': + self._get_cluster_service_subnet(), 'platform::kubernetes::params::apiserver_advertise_address': - self._get_management_address(), + self._get_cluster_host_address(), 'platform::kubernetes::params::etcd_endpoint': self._get_etcd_endpoint(), + 'platform::kubernetes::params::service_domain': + self._get_dns_service_domain(), + 'platform::kubernetes::params::dns_service_ip': + self._get_dns_service_ip(), }) return config @@ -99,7 +103,31 @@ class KubernetesPuppet(base.BasePuppet): return config def _get_etcd_endpoint(self): - addr = self._format_url_address(self._get_management_address()) + addr = self._format_url_address(self._get_cluster_host_address()) protocol = "http" url = "%s://%s:%s" % (protocol, str(addr), str(self.ETCD_SERVICE_PORT)) return url + + def _get_pod_network_cidr(self): + return self._get_network_config(constants.NETWORK_TYPE_CLUSTER_POD) + + def _get_cluster_service_subnet(self): + return self._get_network_config(constants.NETWORK_TYPE_CLUSTER_SERVICE) + + def _get_network_config(self, networktype): + try: + network = self.dbapi.network_get_by_type(networktype) + except exception.NetworkTypeNotFound: + # network not configured + return {} + address_pool = self.dbapi.address_pool_get(network.pool_uuid) + subnet = str(address_pool.network) + '/' + str(address_pool.prefix) + return subnet + + def _get_dns_service_domain(self): + # Setting this to a constant for now. Will be configurable later + return constants.DEFAULT_DNS_SERVICE_DOMAIN + + def _get_dns_service_ip(self): + # Setting this to a constant for now. Will be configurable later + return constants.DEFAULT_DNS_SERVICE_IP diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/networking.py b/sysinv/sysinv/sysinv/sysinv/puppet/networking.py index 96132d91e0..8bb62932bc 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/networking.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/networking.py @@ -22,6 +22,7 @@ class NetworkingPuppet(base.BasePuppet): config.update(self._get_mgmt_network_config()) config.update(self._get_infra_network_config()) config.update(self._get_oam_network_config()) + config.update(self._get_cluster_network_config()) return config def get_host_config(self, host): @@ -29,6 +30,7 @@ class NetworkingPuppet(base.BasePuppet): config.update(self._get_pxeboot_interface_config()) config.update(self._get_mgmt_interface_config()) config.update(self._get_infra_interface_config()) + config.update(self._get_cluster_interface_config()) if host.personality == constants.CONTROLLER: config.update(self._get_oam_interface_config()) return config @@ -88,6 +90,11 @@ class NetworkingPuppet(base.BasePuppet): return config + def _get_cluster_network_config(self): + networktype = constants.NETWORK_TYPE_CLUSTER_HOST + config = self._get_network_config(networktype) + return config + def _get_oam_network_config(self): networktype = constants.NETWORK_TYPE_OAM @@ -147,6 +154,10 @@ class NetworkingPuppet(base.BasePuppet): controller_address_url = self._format_url_address(controller_address) subnet_network_url = self._format_url_address(subnet_network) + # Convert the dash to underscore because puppet parameters cannot have + # dashes + networktype = networktype.replace('-', '_') + return { 'platform::network::%s::params::subnet_version' % networktype: subnet_version, @@ -184,6 +195,9 @@ class NetworkingPuppet(base.BasePuppet): def _get_oam_interface_config(self): return self._get_interface_config(constants.NETWORK_TYPE_OAM) + def _get_cluster_interface_config(self): + return self._get_interface_config(constants.NETWORK_TYPE_CLUSTER_HOST) + def _get_interface_config(self, networktype): config = {} @@ -195,6 +209,9 @@ class NetworkingPuppet(base.BasePuppet): self.context, network_interface) network_id = interface.find_network_id_by_networktype( self.context, networktype) + # Convert the dash to underscore because puppet parameters cannot + # have dashes + networktype = networktype.replace('-', '_') config.update({ 'platform::network::%s::params::interface_name' % networktype: interface_name diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/platform.py b/sysinv/sysinv/sysinv/sysinv/puppet/platform.py index 93ccf4f6a7..0210470927 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/platform.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/platform.py @@ -169,6 +169,23 @@ class PlatformPuppet(base.BasePuppet): (constants.CONTROLLER_CGCS_NFS, constants.CONTROLLER_CGCS_NFS, constants.NETWORK_TYPE_INFRA), + + # cluster network hosts + (constants.CONTROLLER_0_HOSTNAME + HOSTNAME_INFRA_SUFFIX, + constants.CONTROLLER_0_HOSTNAME, + constants.NETWORK_TYPE_CLUSTER_HOST), + + (constants.CONTROLLER_1_HOSTNAME + HOSTNAME_INFRA_SUFFIX, + constants.CONTROLLER_1_HOSTNAME, + constants.NETWORK_TYPE_CLUSTER_HOST), + + (constants.STORAGE_0_HOSTNAME + HOSTNAME_INFRA_SUFFIX, + constants.STORAGE_0_HOSTNAME, + constants.NETWORK_TYPE_CLUSTER_HOST), + + (constants.STORAGE_1_HOSTNAME + HOSTNAME_INFRA_SUFFIX, + constants.STORAGE_1_HOSTNAME, + constants.NETWORK_TYPE_CLUSTER_HOST), ] hosts = {}