From 611a68a96ab915dc4e97d39dffa5c379bbffef3d Mon Sep 17 00:00:00 2001 From: Mingyuan Qi Date: Wed, 30 Jan 2019 09:41:27 +0800 Subject: [PATCH] Allow user specified registries for config_controller Currently docker images were pulled from public registries during config_controller. For some users, the connection to the public docker registry may be slow such that installing the containerized services images may timeout or the system simply does not have access to the public internet. This change allows users to specify alternative public/private registries to replace k8s.gcr.io, gcr.io, quay.io and docker.io. Insecure registry is supported if all default registries were replaced by one unified registry. It lowers the complexity for those who build his own registry without internet access. Docker doesn't support ipv6 addr as registry name, instead hostname or domain name in ipv6 network is allowed. Test: AIO-SX/AIO-DX/Standard(2+2): Alternative public registry (ipv4/domain) with proxy - config_controller pass Private registry (ipv4/ipv6/domain) without internet - config_controller pass Default registry with/without proxy - config_controller pass Story: 2004711 Task: 28742 Change-Id: I4fee3f4e0637863b9b5ef4ef556082ac75f62a1d Signed-off-by: Mingyuan Qi --- .../configutilities/common/utils.py | 34 +- .../configutilities/common/validator.py | 74 +++++ .../controllerconfig/configassistant.py | 290 ++++++++++++++++-- .../tests/files/cgcs_config.kubernetes | 8 + .../tests/files/system_config.kubernetes | 8 + .../tests/test_system_config.py | 7 + .../src/modules/platform/manifests/docker.pp | 5 + .../platform/manifests/dockerdistribution.pp | 18 ++ .../src/modules/platform/manifests/helm.pp | 19 +- .../modules/platform/manifests/kubernetes.pp | 36 ++- .../platform/templates/calico.yaml.erb | 6 +- .../templates/insecuredockerregistry.conf.erb | 2 +- .../platform/templates/kubeadm.yaml.erb | 3 + .../platform/templates/kubelet.conf.erb | 2 + .../src/sysinv/manifests/init.pp | 9 +- .../sysinv/sysinv/sysinv/common/constants.py | 6 + .../sysinv/sysinv/common/service_parameter.py | 57 +++- sysinv/sysinv/sysinv/sysinv/common/utils.py | 27 ++ 18 files changed, 569 insertions(+), 42 deletions(-) create mode 100644 puppet-manifests/src/modules/platform/templates/kubelet.conf.erb diff --git a/configutilities/configutilities/configutilities/common/utils.py b/configutilities/configutilities/configutilities/common/utils.py index e178c533da..ffe03847f1 100644 --- a/configutilities/configutilities/configutilities/common/utils.py +++ b/configutilities/configutilities/configutilities/common/utils.py @@ -182,16 +182,38 @@ def is_valid_ipv6(address): def is_valid_domain_or_ip(url_str): - if is_valid_domain(url_str): - return True - elif is_valid_ipv4(url_str): - return True - elif is_valid_ipv6(url_str): - return True + if url_str: + if is_valid_domain(url_str): + return True + ip_with_port = url_str.split(':') + if len(ip_with_port) <= 2: + # check ipv4 or ipv4 with port + return is_valid_ipv4(ip_with_port[0]) + else: + # check ipv6 with port + if '[' in url_str: + try: + bkt_idx = url_str.index(']') + if bkt_idx + 1 == len(url_str): + # brackets without port + return False + else: + return is_valid_ipv6(url_str[1:bkt_idx]) + except Exception: + return False + else: + # check ipv6 without port + return is_valid_ipv6(url_str) else: return False +def is_valid_bool_str(val): + """Check if the provided string is a valid bool string or not.""" + boolstrs = ('true', 'false') + return str(val).lower() in boolstrs + + def validate_address_str(ip_address_str, network): """Determine whether an address is valid.""" try: diff --git a/configutilities/configutilities/configutilities/common/validator.py b/configutilities/configutilities/configutilities/common/validator.py index 976c132cb1..c635439a83 100644 --- a/configutilities/configutilities/configutilities/common/validator.py +++ b/configutilities/configutilities/configutilities/common/validator.py @@ -26,6 +26,7 @@ from configutilities.common.utils import validate_address_str from configutilities.common.utils import validate_nameserver_address_str from configutilities.common.utils import is_valid_url from configutilities.common.utils import is_valid_domain_or_ip +from configutilities.common.utils import is_valid_bool_str from configutilities.common.exceptions import ConfigFail from configutilities.common.exceptions import ValidateFail @@ -1010,6 +1011,77 @@ class ConfigValidator(object): self.cgcs_conf.set('cDOCKER_PROXY', 'DOCKER_NO_PROXY', docker_no_proxy_list_str) + def validate_docker_registry(self): + if not self.conf.has_section('DOCKER_REGISTRY'): + return + if self.cgcs_conf is not None: + self.cgcs_conf.add_section('cDOCKER_REGISTRY') + # check k8s_registry + if self.conf.has_option('DOCKER_REGISTRY', 'DOCKER_K8S_REGISTRY'): + docker_k8s_registry_str = self.conf.get( + 'DOCKER_REGISTRY', 'DOCKER_K8S_REGISTRY') + if is_valid_domain_or_ip(docker_k8s_registry_str): + if self.cgcs_conf is not None: + self.cgcs_conf.set('cDOCKER_REGISTRY', + 'DOCKER_K8S_REGISTRY', + docker_k8s_registry_str) + else: + raise ConfigFail( + "Invalid DOCKER_K8S_REGISTRY value of %s." % + docker_k8s_registry_str) + # check gcr_registry + if self.conf.has_option('DOCKER_REGISTRY', 'DOCKER_GCR_REGISTRY'): + docker_gcr_registry_str = self.conf.get( + 'DOCKER_REGISTRY', 'DOCKER_GCR_REGISTRY') + if is_valid_domain_or_ip(docker_gcr_registry_str): + if self.cgcs_conf is not None: + self.cgcs_conf.set('cDOCKER_REGISTRY', + 'DOCKER_GCR_REGISTRY', + docker_gcr_registry_str) + else: + raise ConfigFail( + "Invalid DOCKER_GCR_REGISTRY value of %s." % + docker_gcr_registry_str) + # check quay_registry + if self.conf.has_option('DOCKER_REGISTRY', 'DOCKER_QUAY_REGISTRY'): + docker_quay_registry_str = self.conf.get( + 'DOCKER_REGISTRY', 'DOCKER_QUAY_REGISTRY') + if is_valid_domain_or_ip(docker_quay_registry_str): + if self.cgcs_conf is not None: + self.cgcs_conf.set('cDOCKER_REGISTRY', + 'DOCKER_QUAY_REGISTRY', + docker_quay_registry_str) + else: + raise ConfigFail( + "Invalid DOCKER_QUAY_REGISTRY value of %s." % + docker_quay_registry_str) + # check docker_registry + if self.conf.has_option('DOCKER_REGISTRY', 'DOCKER_DOCKER_REGISTRY'): + docker_docker_registry_str = self.conf.get( + 'DOCKER_REGISTRY', 'DOCKER_DOCKER_REGISTRY') + if is_valid_domain_or_ip(docker_docker_registry_str): + if self.cgcs_conf is not None: + self.cgcs_conf.set('cDOCKER_REGISTRY', + 'DOCKER_DOCKER_REGISTRY', + docker_docker_registry_str) + else: + raise ConfigFail( + "Invalid DOCKER_DOCKER_REGISTRY value of %s." % + docker_docker_registry_str) + # check is_secure_registry + if self.conf.has_option('DOCKER_REGISTRY', 'IS_SECURE_REGISTRY'): + docker_is_secure_registry_str = self.conf.get( + 'DOCKER_REGISTRY', 'IS_SECURE_REGISTRY') + if is_valid_bool_str(docker_is_secure_registry_str): + if self.cgcs_conf is not None: + self.cgcs_conf.set('cDOCKER_REGISTRY', + 'IS_SECURE_REGISTRY', + docker_is_secure_registry_str) + else: + raise ConfigFail( + "Invalid IS_SECURE_REGISTRY value of %s." % + docker_is_secure_registry_str) + def validate_ntp(self): if self.conf.has_section('NTP'): raise ConfigFail("NTP Configuration is no longer supported") @@ -1468,6 +1540,8 @@ def validate(system_config, config_type=REGION_CONFIG, cgcs_config=None, validator.validate_dns() # Docker Proxy configuration validator.validate_docker_proxy() + # Docker Registry configuration + validator.validate_docker_registry() # NTP configuration validator.validate_ntp() # Network configuration diff --git a/controllerconfig/controllerconfig/controllerconfig/configassistant.py b/controllerconfig/controllerconfig/controllerconfig/configassistant.py index 2915f6b783..e6dd7f5281 100644 --- a/controllerconfig/controllerconfig/controllerconfig/configassistant.py +++ b/controllerconfig/controllerconfig/controllerconfig/configassistant.py @@ -470,6 +470,14 @@ class ConfigAssistant(): self.docker_https_proxy = "" self.docker_no_proxy = "" + # Docker registry config + self.docker_use_default_registry = True + self.docker_k8s_registry = "" + self.docker_gcr_registry = "" + self.docker_quay_registry = "" + self.docker_docker_registry = "" + self.is_secure_registry = True + # SDN config self.enable_sdn = False @@ -2741,7 +2749,7 @@ class ConfigAssistant(): """Allow user to input docker proxy config.""" print("\nDocker Proxy:") - print("-------------------------\n") + print("-------------\n") print(textwrap.fill( "Docker proxy is needed if host OAM network is behind a proxy.", 80)) @@ -2762,7 +2770,7 @@ class ConfigAssistant(): self.docker_http_proxy = user_input break else: - print("Please input a valid url") + print("Please enter a valid url") continue else: self.docker_http_proxy = "" @@ -2778,7 +2786,7 @@ class ConfigAssistant(): self.docker_https_proxy = user_input break else: - print("Please input a valid url") + print("Please enter a valid url") continue else: self.docker_https_proxy = "" @@ -2849,6 +2857,145 @@ class ConfigAssistant(): print("Invalid choice") continue + def input_docker_registry_config(self): + """Allow user to input docker registry config.""" + + print("\nDocker Registry:") + print("----------------\n") + print("Configure docker registries to pull images from.\n" + "Default registries are:\n" + "k8s.gcr.io, gcr.io, quay.io, docker.io\n" + ) + while True: + user_input = input( + "Use default docker registries [Y/n]: ") + if user_input.lower() == 'q': + raise UserQuit + elif user_input.lower() == 'n': + # unexpected newline displayed if textwrap.fill with + # '\n' included + print("\nEach registry can be specified as one of the" + "following:\n" + " - domain (e.g. example.domain)\n" + " - domain with port (e.g. example.domain:5000)\n" + " - IPv4 address (e.g. 1.2.3.4)\n" + " - IPv4 address with port (e.g. 1.2.3.4:5000)\n" + ) + while True: + user_input = input( + "Use a unified registry replacing all " + "default registries [y/n]: ") + if user_input.lower() == 'q': + raise UserQuit + elif user_input.lower() == 'y': + # Input a unified registry to avoid + # inputing the same registry repeatly + while True: + user_input = input( + "Enter a unified registry: ") + if user_input.lower() == 'q': + raise UserQuit + if is_valid_domain_or_ip(user_input): + self.docker_k8s_registry = user_input + self.docker_gcr_registry = user_input + self.docker_quay_registry = user_input + self.docker_docker_registry = user_input + self.docker_use_default_registry = False + break + else: + print("Please enter a valid registry address") + continue + + # Only if a unified registry set, it could be + # an insecure registry + while True: + user_input = input( + "Is '" + self.docker_k8s_registry + + "' a secure registry (https) [Y/n]: ") + if user_input.lower() == 'q': + raise UserQuit + elif user_input.lower() in ('y', ''): + self.is_secure_registry = True + break + elif user_input.lower() == 'n': + self.is_secure_registry = False + break + else: + print("Invalid choice") + continue + break + + elif user_input.lower() == 'n': + # Input alternative registries separately + while True: + user_input = input( + "Alternative registry to k8s.gcr.io: ") + if user_input.lower() == 'q': + raise UserQuit + if is_valid_domain_or_ip(user_input): + self.docker_k8s_registry = user_input + break + else: + print("Please enter a valid registry address") + continue + + while True: + user_input = input( + "Alternative registry to gcr.io: ") + if user_input.lower() == 'q': + raise UserQuit + if is_valid_domain_or_ip(user_input): + self.docker_gcr_registry = user_input + break + else: + print("Please enter a valid registry address") + continue + + while True: + user_input = input( + "Alternative registry to quay.io: ") + if user_input.lower() == 'q': + raise UserQuit + if is_valid_domain_or_ip(user_input): + self.docker_quay_registry = user_input + break + else: + print("Please enter a valid registry address") + continue + + while True: + user_input = input( + "Alternative registry to docker.io: ") + if user_input.lower() == 'q': + raise UserQuit + if is_valid_domain_or_ip(user_input): + self.docker_docker_registry = user_input + break + else: + print("Please enter a valid registry address") + continue + + if (self.docker_k8s_registry or + self.docker_gcr_registry or + self.docker_quay_registry or + self.docker_docker_registry): + self.docker_use_default_registry = False + break + else: + print("At least one registry is required") + continue + else: + print("Invalid choice") + continue + break + + elif user_input.lower() in ('y', ''): + self.docker_use_default_registry = True + break + else: + print("Invalid choice") + continue + def input_authentication_config(self): """Allow user to input authentication config and perform validation. """ @@ -2936,6 +3083,7 @@ class ConfigAssistant(): self.input_dns_config() # Docker proxy is only used in kubernetes config self.input_docker_proxy_config() + self.input_docker_registry_config() self.input_authentication_config() def is_valid_management_multicast_subnet(self, ip_subnet): @@ -3298,6 +3446,32 @@ class ConfigAssistant(): self.docker_no_proxy = config.get( 'cDOCKER_PROXY', 'DOCKER_NO_PROXY') + # Docker Registry Configuration + if config.has_section('cDOCKER_REGISTRY'): + self.docker_use_default_registry = False + if config.has_option('cDOCKER_REGISTRY', + 'DOCKER_K8S_REGISTRY'): + self.docker_k8s_registry = config.get( + 'cDOCKER_REGISTRY', 'DOCKER_K8S_REGISTRY') + if config.has_option('cDOCKER_REGISTRY', + 'DOCKER_GCR_REGISTRY'): + self.docker_gcr_registry = config.get( + 'cDOCKER_REGISTRY', 'DOCKER_GCR_REGISTRY') + if config.has_option('cDOCKER_REGISTRY', + 'DOCKER_QUAY_REGISTRY'): + self.docker_quay_registry = config.get( + 'cDOCKER_REGISTRY', 'DOCKER_QUAY_REGISTRY') + if config.has_option('cDOCKER_REGISTRY', + 'DOCKER_DOCKER_REGISTRY'): + self.docker_docker_registry = config.get( + 'cDOCKER_REGISTRY', 'DOCKER_DOCKER_REGISTRY') + if config.has_option('cDOCKER_REGISTRY', + 'IS_SECURE_REGISTRY'): + self.is_secure_registry = config.getboolean( + 'cDOCKER_REGISTRY', 'IS_SECURE_REGISTRY') + else: + self.is_secure_registry = True + # SDN Network configuration if config.has_option('cSDN', 'ENABLE_SDN'): raise ConfigFail("The option ENABLE_SDN is no longer " @@ -3742,14 +3916,31 @@ class ConfigAssistant(): if not dns_config: print("External DNS servers not configured") if self.enable_docker_proxy: - print("\nDocker Proxy Configuraton") - print("----------------------") + print("\nDocker Proxy Configuration") + print("--------------------------") if self.docker_http_proxy: print("Docker HTTP proxy: " + self.docker_http_proxy) if self.docker_https_proxy: print("Docker HTTPS proxy: " + self.docker_https_proxy) if self.docker_no_proxy: print("Docker NO proxy: " + self.docker_no_proxy) + if not self.docker_use_default_registry: + print("\nDocker Registry Configuration") + print("-----------------------------") + if self.docker_k8s_registry: + print("Alternative registry to k8s.gcr.io: " + + self.docker_k8s_registry) + if self.docker_gcr_registry: + print("Alternative registry to gcr.io: " + + self.docker_gcr_registry) + if self.docker_quay_registry: + print("Alternative registry to quay.io: " + + self.docker_quay_registry) + if self.docker_docker_registry: + print("Alternative registry to docker.io: " + + self.docker_docker_registry) + print("Is registries secure: " + + str(self.is_secure_registry)) if self.region_config: print("\nRegion Configuration") @@ -4064,6 +4255,30 @@ class ConfigAssistant(): "DOCKER_NO_PROXY=" + str(self.docker_no_proxy) + "\n") + # Docker registry configuration + if not self.docker_use_default_registry: + f.write("\n[cDOCKER_REGISTRY]") + f.write("\n# Docker Registry Configuration\n") + if self.docker_k8s_registry: + f.write( + "DOCKER_K8S_REGISTRY=" + + str(self.docker_k8s_registry) + "\n") + if self.docker_gcr_registry: + f.write( + "DOCKER_GCR_REGISTRY=" + + str(self.docker_gcr_registry) + "\n") + if self.docker_quay_registry: + f.write( + "DOCKER_QUAY_REGISTRY=" + + str(self.docker_quay_registry) + "\n") + if self.docker_docker_registry: + f.write( + "DOCKER_DOCKER_REGISTRY=" + + str(self.docker_docker_registry) + "\n") + f.write( + "IS_SECURE_REGISTRY=" + + str(self.is_secure_registry) + "\n") + # Network configuration f.write("\n[cNETWORK]") f.write("\n# Data Network Configuration\n") @@ -5468,22 +5683,54 @@ class ConfigAssistant(): client.sysinv.idns.update(dns_record.uuid, patch) def _populate_docker_config(self, client): - parameter = {} - if self.docker_http_proxy: - parameter['http_proxy'] = self.docker_http_proxy - if self.docker_https_proxy: - parameter['https_proxy'] = self.docker_https_proxy - if self.docker_no_proxy: - parameter['no_proxy'] = self.docker_no_proxy + if self.enable_docker_proxy: + proxy_parameter = {} + if self.docker_http_proxy: + proxy_parameter['http_proxy'] = self.docker_http_proxy + if self.docker_https_proxy: + proxy_parameter['https_proxy'] = self.docker_https_proxy + if self.docker_no_proxy: + proxy_parameter['no_proxy'] = self.docker_no_proxy - if parameter: - client.sysinv.service_parameter.create( - sysinv_constants.SERVICE_TYPE_DOCKER, - sysinv_constants.SERVICE_PARAM_SECTION_DOCKER_PROXY, - None, - None, - parameter - ) + if proxy_parameter: + client.sysinv.service_parameter.create( + sysinv_constants.SERVICE_TYPE_DOCKER, + sysinv_constants.SERVICE_PARAM_SECTION_DOCKER_PROXY, + None, + None, + proxy_parameter + ) + + if not self.docker_use_default_registry: + registry_parameter = {} + if self.docker_k8s_registry: + registry_parameter['k8s'] = \ + self.docker_k8s_registry + + if self.docker_gcr_registry: + registry_parameter['gcr'] = \ + self.docker_gcr_registry + + if self.docker_quay_registry: + registry_parameter['quay'] = \ + self.docker_quay_registry + + if self.docker_docker_registry: + registry_parameter['docker'] = \ + self.docker_docker_registry + + if not self.is_secure_registry: + registry_parameter['insecure_registry'] = "True" + + if registry_parameter: + client.sysinv.service_parameter.create( + sysinv_constants.SERVICE_TYPE_DOCKER, + sysinv_constants. + SERVICE_PARAM_SECTION_DOCKER_REGISTRY, + None, + None, + registry_parameter + ) def populate_initial_config(self): """Populate initial system inventory configuration""" @@ -5494,8 +5741,7 @@ class ConfigAssistant(): self._populate_network_config(client) if self.kubernetes: self._populate_dns_config(client) - if self.enable_docker_proxy: - self._populate_docker_config(client) + self._populate_docker_config(client) controller = self._populate_controller_config(client) # ceph_mon config requires controller host to be created self._inventory_config_complete_wait(client, controller) diff --git a/controllerconfig/controllerconfig/controllerconfig/tests/files/cgcs_config.kubernetes b/controllerconfig/controllerconfig/controllerconfig/tests/files/cgcs_config.kubernetes index 7df235ba86..2f959d5d5e 100755 --- a/controllerconfig/controllerconfig/controllerconfig/tests/files/cgcs_config.kubernetes +++ b/controllerconfig/controllerconfig/controllerconfig/tests/files/cgcs_config.kubernetes @@ -79,6 +79,14 @@ DOCKER_HTTP_PROXY=http://proxy.com:123 DOCKER_HTTPS_PROXY=https://proxy.com:123 DOCKER_NO_PROXY=localhost,127.0.0.1,192.168.204.2 +[cDOCKER_REGISTRY] +# Docker Registry Configuration +DOCKER_K8S_REGISTRY=my.registry.com:5000 +DOCKER_GCR_REGISTRY=my.registry.com +DOCKER_QUAY_REGISTRY=1.2.3.4:5000 +DOCKER_DOCKER_REGISTRY=[1:2:3:4:a:b:c:d]:5000 +IS_SECURE_REGISTRY=False + [cNETWORK] # Data Network Configuration VSWITCH_TYPE=ovs-dpdk diff --git a/controllerconfig/controllerconfig/controllerconfig/tests/files/system_config.kubernetes b/controllerconfig/controllerconfig/controllerconfig/tests/files/system_config.kubernetes index 7ff90974ac..0a1d209714 100755 --- a/controllerconfig/controllerconfig/controllerconfig/tests/files/system_config.kubernetes +++ b/controllerconfig/controllerconfig/controllerconfig/tests/files/system_config.kubernetes @@ -55,6 +55,14 @@ DOCKER_HTTP_PROXY=http://proxy.com:123 DOCKER_HTTPS_PROXY=https://proxy.com:123 DOCKER_NO_PROXY=localhost,127.0.0.1,192.168.204.2 +[DOCKER_REGISTRY] +# Docker Registry Configuration +DOCKER_K8S_REGISTRY=my.registry.com:5000 +DOCKER_GCR_REGISTRY=my.registry.com +DOCKER_QUAY_REGISTRY=1.2.3.4:5000 +DOCKER_DOCKER_REGISTRY=[1:2:3:4:a:b:c:d]:5000 +IS_SECURE_REGISTRY=False + ;[PXEBOOT_NETWORK] ;PXEBOOT_CIDR=192.168.203.0/24 diff --git a/controllerconfig/controllerconfig/controllerconfig/tests/test_system_config.py b/controllerconfig/controllerconfig/controllerconfig/tests/test_system_config.py index a8438eb8ee..2bda2825c9 100644 --- a/controllerconfig/controllerconfig/controllerconfig/tests/test_system_config.py +++ b/controllerconfig/controllerconfig/controllerconfig/tests/test_system_config.py @@ -654,3 +654,10 @@ def test_kubernetes(): cr.create_cgcs_config_file(None, system_config, None, None, None, 0, validate_only=True) validate(system_config, DEFAULT_CONFIG, None, False) + + # Test absence of optional docker registry configuration + system_config = cr.parse_system_config(systemfile) + system_config.remove_section('DOCKER_REGISTRY') + cr.create_cgcs_config_file(None, system_config, None, None, None, 0, + validate_only=True) + validate(system_config, DEFAULT_CONFIG, None, False) diff --git a/puppet-manifests/src/modules/platform/manifests/docker.pp b/puppet-manifests/src/modules/platform/manifests/docker.pp index 23beb2e884..2e67dbf113 100644 --- a/puppet-manifests/src/modules/platform/manifests/docker.pp +++ b/puppet-manifests/src/modules/platform/manifests/docker.pp @@ -3,6 +3,11 @@ class platform::docker::params ( $http_proxy = undef, $https_proxy = undef, $no_proxy = undef, + $k8s_registry = undef, + $gcr_registry = undef, + $quay_registry = undef, + $docker_registry = undef, + $insecure_registry = undef, ) { } class platform::docker::config diff --git a/puppet-manifests/src/modules/platform/manifests/dockerdistribution.pp b/puppet-manifests/src/modules/platform/manifests/dockerdistribution.pp index 6aaecee5bd..453ab7dadd 100644 --- a/puppet-manifests/src/modules/platform/manifests/dockerdistribution.pp +++ b/puppet-manifests/src/modules/platform/manifests/dockerdistribution.pp @@ -7,9 +7,18 @@ class platform::dockerdistribution::config if $enabled { include ::platform::network::mgmt::params + include ::platform::docker::params $docker_registry_ip = $::platform::network::mgmt::params::controller_address + # check insecure registries + if $::platform::docker::params::insecure_registry { + # insecure registry is true means unified registry was set + $insecure_registries = "\"${::platform::docker::params::k8s_registry}\", \"${docker_registry_ip}:9001\"" + } else { + $insecure_registries = "\"${docker_registry_ip}:9001\"" + } + # currently docker registry is running insecure mode # when proper authentication is implemented, this would go away file { '/etc/docker': @@ -52,9 +61,18 @@ class platform::dockerdistribution::compute $enabled = $::platform::kubernetes::params::enabled if $enabled { include ::platform::network::mgmt::params + include ::platform::docker::params $docker_registry_ip = $::platform::network::mgmt::params::controller_address + # check insecure registries + if $::platform::docker::params::insecure_registry { + # insecure registry is true means unified registry was set + $insecure_registries = "\"${::platform::docker::params::k8s_registry}\", \"${docker_registry_ip}:9001\"" + } else { + $insecure_registries = "\"${docker_registry_ip}:9001\"" + } + # currently docker registry is running insecure mode # when proper authentication is implemented, this would go away file { '/etc/docker': diff --git a/puppet-manifests/src/modules/platform/manifests/helm.pp b/puppet-manifests/src/modules/platform/manifests/helm.pp index 31396dae90..f92c54c487 100644 --- a/puppet-manifests/src/modules/platform/manifests/helm.pp +++ b/puppet-manifests/src/modules/platform/manifests/helm.pp @@ -7,6 +7,7 @@ class platform::helm inherits ::platform::helm::repository::params { include ::platform::kubernetes::params + include ::platform::docker::params if $::platform::kubernetes::params::enabled { file {$source_helm_repo_dir: @@ -27,17 +28,29 @@ class platform::helm if str2bool($::is_initial_config_primary) { + if $::platform::docker::params::gcr_registry { + $gcr_registry = $::platform::docker::params::gcr_registry + } else { + $gcr_registry = 'gcr.io' + } + + if $::platform::docker::params::quay_registry { + $quay_registry = $::platform::docker::params::quay_registry + } else { + $quay_registry = 'quay.io' + } + Class['::platform::kubernetes::master'] # TODO(jrichard): Upversion tiller image to v2.11.1 once released. -> exec { 'load tiller docker image': - command => 'docker image pull gcr.io/kubernetes-helm/tiller:v2.12.1', + command => "docker image pull ${gcr_registry}/kubernetes-helm/tiller:v2.12.1", logoutput => true, } # TODO(tngo): If and when tiller image is upversioned, please ensure armada compatibility as part of the test -> exec { 'load armada docker image': - command => 'docker image pull quay.io/airshipit/armada:f807c3a1ec727c883c772ffc618f084d960ed5c9', + command => "docker image pull ${quay_registry}/airshipit/armada:f807c3a1ec727c883c772ffc618f084d960ed5c9", logoutput => true, } @@ -54,7 +67,7 @@ class platform::helm # TODO(jrichard): Upversion tiller image to v2.11.1 once released. -> exec { 'initialize helm': environment => [ 'KUBECONFIG=/etc/kubernetes/admin.conf', 'HOME=/home/wrsroot' ], - command => 'helm init --skip-refresh --service-account tiller --node-selectors "node-role.kubernetes.io/master"="" --tiller-image=gcr.io/kubernetes-helm/tiller:v2.12.1', # lint:ignore:140chars + command => "helm init --skip-refresh --service-account tiller --node-selectors \"node-role.kubernetes.io/master\"=\"\" --tiller-image=${gcr_registry}/kubernetes-helm/tiller:v2.12.1", # lint:ignore:140chars logoutput => true, user => 'wrsroot', group => 'wrs', diff --git a/puppet-manifests/src/modules/platform/manifests/kubernetes.pp b/puppet-manifests/src/modules/platform/manifests/kubernetes.pp index 49c28fdba9..e68b852b44 100644 --- a/puppet-manifests/src/modules/platform/manifests/kubernetes.pp +++ b/puppet-manifests/src/modules/platform/manifests/kubernetes.pp @@ -13,9 +13,27 @@ class platform::kubernetes::params ( ) { } class platform::kubernetes::kubeadm { + include ::platform::docker::params + $iptables_file = "net.bridge.bridge-nf-call-ip6tables = 1 net.bridge.bridge-nf-call-iptables = 1" + if $::platform::docker::params::k8s_registry { + $k8s_registry = $::platform::docker::params::k8s_registry + } else { + $k8s_registry = undef + } + + # kubelet use --pod-infra-container-image to indentify the specified image + # TODO: this is not needed after kubernetes upgraded to 1.13 + # because the imageRepository setting will be used + if $k8s_registry { + file { '/etc/sysconfig/kubelet': + ensure => file, + content => template('platform/kubelet.conf.erb'), + } + } + # Update iptables config. This is required based on: # https://kubernetes.io/docs/tasks/tools/install-kubeadm # This probably belongs somewhere else - initscripts package? @@ -52,6 +70,21 @@ class platform::kubernetes::master::init inherits ::platform::kubernetes::params { include ::platform::params + include ::platform::docker::params + + # This is used for imageRepository in template kubeadm.yaml.erb + if $::platform::docker::params::k8s_registry { + $k8s_registry = $::platform::docker::params::k8s_registry + } else { + $k8s_registry = undef + } + + # This is used for calico image in template calico.yaml.erb + if $::platform::docker::params::quay_registry { + $quay_registry = $::platform::docker::params::quay_registry + } else { + $quay_registry = 'quay.io' + } if str2bool($::is_initial_config_primary) { # For initial controller install, configure kubernetes from scratch. @@ -347,10 +380,11 @@ class platform::kubernetes::worker } # TODO: remove port 9001 once we have a public docker image registry using standard ports. +# add 5000 as the default port for private registry class platform::kubernetes::firewall::params ( $transport = 'tcp', $table = 'nat', - $dports = [80, 443, 9001], + $dports = [80, 443, 9001, 5000], $chain = 'POSTROUTING', $jump = 'SNAT', ) {} diff --git a/puppet-manifests/src/modules/platform/templates/calico.yaml.erb b/puppet-manifests/src/modules/platform/templates/calico.yaml.erb index 1f6eea5ffc..6c6a439cf6 100644 --- a/puppet-manifests/src/modules/platform/templates/calico.yaml.erb +++ b/puppet-manifests/src/modules/platform/templates/calico.yaml.erb @@ -107,7 +107,7 @@ spec: # as a host-networked pod. serviceAccountName: calico-node containers: - - image: quay.io/calico/typha:v3.1.4 + - image: <%= @quay_registry %>/calico/typha:v3.1.4 name: calico-typha ports: - containerPort: 5473 @@ -198,7 +198,7 @@ spec: # container programs network policy and routes on each # host. - name: calico-node - image: quay.io/calico/node:v3.1.4 + image: <%= @quay_registry %>/calico/node:v3.1.4 env: # Use Kubernetes API as the backing datastore. - name: DATASTORE_TYPE @@ -283,7 +283,7 @@ spec: # This container installs the Calico CNI binaries # and CNI network config file on each node. - name: install-cni - image: quay.io/calico/cni:v3.1.4 + image: <%= @quay_registry %>/calico/cni:v3.1.4 command: ["/install-cni.sh"] env: # Name of the CNI config file to create. diff --git a/puppet-manifests/src/modules/platform/templates/insecuredockerregistry.conf.erb b/puppet-manifests/src/modules/platform/templates/insecuredockerregistry.conf.erb index 0998bc827f..911fc31300 100644 --- a/puppet-manifests/src/modules/platform/templates/insecuredockerregistry.conf.erb +++ b/puppet-manifests/src/modules/platform/templates/insecuredockerregistry.conf.erb @@ -1,3 +1,3 @@ { - "insecure-registries" : [ "<%= @docker_registry_ip %>:9001" ] + "insecure-registries" : [ <%= @insecure_registries %> ] } diff --git a/puppet-manifests/src/modules/platform/templates/kubeadm.yaml.erb b/puppet-manifests/src/modules/platform/templates/kubeadm.yaml.erb index c64b03471b..32efb4b615 100644 --- a/puppet-manifests/src/modules/platform/templates/kubeadm.yaml.erb +++ b/puppet-manifests/src/modules/platform/templates/kubeadm.yaml.erb @@ -21,6 +21,9 @@ controllerManagerExtraArgs: node-monitor-period: "2s" node-monitor-grace-period: "20s" pod-eviction-timeout: "30s" +<%- if @k8s_registry -%> +imageRepository: "<%= @k8s_registry %>" +<%- end -%> --- kind: KubeletConfiguration apiVersion: kubelet.config.k8s.io/v1beta1 diff --git a/puppet-manifests/src/modules/platform/templates/kubelet.conf.erb b/puppet-manifests/src/modules/platform/templates/kubelet.conf.erb new file mode 100644 index 0000000000..6c36049dfd --- /dev/null +++ b/puppet-manifests/src/modules/platform/templates/kubelet.conf.erb @@ -0,0 +1,2 @@ +# Overrides config file for kubelet +KUBELET_EXTRA_ARGS=--pod-infra-container-image=<%= @k8s_registry %>/pause:3.1 diff --git a/puppet-modules-wrs/puppet-sysinv/src/sysinv/manifests/init.pp b/puppet-modules-wrs/puppet-sysinv/src/sysinv/manifests/init.pp index 2fa57491e6..9dd51a8add 100644 --- a/puppet-modules-wrs/puppet-sysinv/src/sysinv/manifests/init.pp +++ b/puppet-modules-wrs/puppet-sysinv/src/sysinv/manifests/init.pp @@ -77,6 +77,7 @@ class sysinv ( include sysinv::params include ::platform::kubernetes::params + include ::platform::docker::params Package['sysinv'] -> Sysinv_config<||> Package['sysinv'] -> Sysinv_api_paste_ini<||> @@ -217,7 +218,13 @@ class sysinv ( } if $::platform::kubernetes::params::enabled == true { - $armada_img_tag = 'quay.io/airshipit/armada:f807c3a1ec727c883c772ffc618f084d960ed5c9' + if $::platform::docker::params::quay_registry { + $quay_registry = $::platform::docker::params::quay_registry + } else { + $quay_registry = 'quay.io' + } + + $armada_img_tag = "${quay_registry}/airshipit/armada:f807c3a1ec727c883c772ffc618f084d960ed5c9" sysinv_config { 'DEFAULT/armada_image_tag': value => $armada_img_tag; } diff --git a/sysinv/sysinv/sysinv/sysinv/common/constants.py b/sysinv/sysinv/sysinv/sysinv/common/constants.py index 682612e29d..132dbdfa9a 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/constants.py +++ b/sysinv/sysinv/sysinv/sysinv/common/constants.py @@ -1103,6 +1103,12 @@ SERVICE_PARAM_SECTION_DOCKER_PROXY = 'proxy' SERVICE_PARAM_NAME_DOCKER_HTTP_PROXY = 'http_proxy' SERVICE_PARAM_NAME_DOCKER_HTTPS_PROXY = 'https_proxy' SERVICE_PARAM_NAME_DOCKER_NO_PROXY = 'no_proxy' +SERVICE_PARAM_SECTION_DOCKER_REGISTRY = 'registry' +SERVICE_PARAM_NAME_DOCKER_K8S_REGISTRY = 'k8s' +SERVICE_PARAM_NAME_DOCKER_GCR_REGISTRY = 'gcr' +SERVICE_PARAM_NAME_DOCKER_QUAY_REGISTRY = 'quay' +SERVICE_PARAM_NAME_DOCKER_DOCKER_REGISTRY = 'docker' +SERVICE_PARAM_NAME_DOCKER_INSECURE_REGISTRY = 'insecure_registry' # default filesystem size to 25 MB SERVICE_PARAM_SWIFT_FS_SIZE_MB_DEFAULT = 25 diff --git a/sysinv/sysinv/sysinv/sysinv/common/service_parameter.py b/sysinv/sysinv/sysinv/sysinv/common/service_parameter.py index 1dac250187..8d7c014a05 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/service_parameter.py +++ b/sysinv/sysinv/sysinv/sysinv/common/service_parameter.py @@ -660,11 +660,24 @@ def _validate_docker_no_proxy_address(name, value): values = value.split(',') for item in values: # will extend to more cases if CIDR notation is supported - if not cutils.is_valid_domain(item): - if not cutils.is_valid_ip(item): - raise wsme.exc.ClientSideError(_( - "Parameter '%s' includes an invalid address '%s'." % - (name, item))) + if not cutils.is_valid_domain_or_ip(item): + raise wsme.exc.ClientSideError(_( + "Parameter '%s' includes an invalid address '%s'." % + (name, item))) + + +def _validate_docker_registry_address(name, value): + """Check if registry address is valid""" + if not cutils.is_valid_domain_or_ip(value): + raise wsme.exc.ClientSideError(_( + "Parameter '%s' must be a valid address." % name)) + + +def _validate_docker_insecure_registry_bool(name, value): + """Check if insecure registry is a valid bool""" + if not cutils.is_valid_boolstr(value): + raise wsme.exc.ClientSideError(_( + "Parameter '%s' must be a valid bool string." % name)) # LDAP Identity Service Parameters (mandatory) @@ -1516,6 +1529,35 @@ DOCKER_PROXY_PARAMETER_RESOURCE = { 'platform::docker::params::no_proxy', } +DOCKER_REGISTRY_PARAMETER_OPTIONAL = [ + constants.SERVICE_PARAM_NAME_DOCKER_K8S_REGISTRY, + constants.SERVICE_PARAM_NAME_DOCKER_GCR_REGISTRY, + constants.SERVICE_PARAM_NAME_DOCKER_QUAY_REGISTRY, + constants.SERVICE_PARAM_NAME_DOCKER_DOCKER_REGISTRY, + constants.SERVICE_PARAM_NAME_DOCKER_INSECURE_REGISTRY, +] + +DOCKER_REGISTRY_PARAMETER_VALIDATOR = { + constants.SERVICE_PARAM_NAME_DOCKER_K8S_REGISTRY: _validate_docker_registry_address, + constants.SERVICE_PARAM_NAME_DOCKER_GCR_REGISTRY: _validate_docker_registry_address, + constants.SERVICE_PARAM_NAME_DOCKER_QUAY_REGISTRY: _validate_docker_registry_address, + constants.SERVICE_PARAM_NAME_DOCKER_DOCKER_REGISTRY: _validate_docker_registry_address, + constants.SERVICE_PARAM_NAME_DOCKER_INSECURE_REGISTRY: _validate_docker_insecure_registry_bool, +} + +DOCKER_REGISTRY_PARAMETER_RESOURCE = { + constants.SERVICE_PARAM_NAME_DOCKER_K8S_REGISTRY: + 'platform::docker::params::k8s_registry', + constants.SERVICE_PARAM_NAME_DOCKER_GCR_REGISTRY: + 'platform::docker::params::gcr_registry', + constants.SERVICE_PARAM_NAME_DOCKER_QUAY_REGISTRY: + 'platform::docker::params::quay_registry', + constants.SERVICE_PARAM_NAME_DOCKER_DOCKER_REGISTRY: + 'platform::docker::params::docker_registry', + constants.SERVICE_PARAM_NAME_DOCKER_INSECURE_REGISTRY: + 'platform::docker::params::insecure_registry', +} + HTTPD_PORT_PARAMETER_OPTIONAL = [ constants.SERVICE_PARAM_HTTP_PORT_HTTP, constants.SERVICE_PARAM_HTTP_PORT_HTTPS, @@ -1715,6 +1757,11 @@ SERVICE_PARAMETER_SCHEMA = { SERVICE_PARAM_VALIDATOR: DOCKER_PROXY_PARAMETER_VALIDATOR, SERVICE_PARAM_RESOURCE: DOCKER_PROXY_PARAMETER_RESOURCE, }, + constants.SERVICE_PARAM_SECTION_DOCKER_REGISTRY: { + SERVICE_PARAM_OPTIONAL: DOCKER_REGISTRY_PARAMETER_OPTIONAL, + SERVICE_PARAM_VALIDATOR: DOCKER_REGISTRY_PARAMETER_VALIDATOR, + SERVICE_PARAM_RESOURCE: DOCKER_REGISTRY_PARAMETER_RESOURCE, + }, }, constants.SERVICE_TYPE_HTTP: { constants.SERVICE_PARAM_SECTION_HTTP_CONFIG: { diff --git a/sysinv/sysinv/sysinv/sysinv/common/utils.py b/sysinv/sysinv/sysinv/sysinv/common/utils.py index a87ebc65d4..12d2be41c3 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/utils.py +++ b/sysinv/sysinv/sysinv/sysinv/common/utils.py @@ -1832,6 +1832,33 @@ def is_valid_domain(url_str): return False +def is_valid_domain_or_ip(url_str): + if url_str: + if is_valid_domain(url_str): + return True + ip_with_port = url_str.split(':') + if len(ip_with_port) <= 2: + # check ipv4 or ipv4 with port + return is_valid_ipv4(ip_with_port[0]) + else: + # check ipv6 with port + if '[' in url_str: + try: + bkt_idx = url_str.index(']') + if bkt_idx + 1 == len(url_str): + # brackets without port + return False + else: + return is_valid_ipv6(url_str[1:bkt_idx]) + except Exception: + return False + else: + # check ipv6 without port + return is_valid_ipv6(url_str) + else: + return False + + def verify_checksum(path): """ Find and validate the checksum file in a given directory. """ rc = True