diff --git a/configutilities/configutilities/configutilities/common/utils.py b/configutilities/configutilities/configutilities/common/utils.py index e178c533da..ffe03847f1 100644 --- a/configutilities/configutilities/configutilities/common/utils.py +++ b/configutilities/configutilities/configutilities/common/utils.py @@ -182,16 +182,38 @@ def is_valid_ipv6(address): def is_valid_domain_or_ip(url_str): - if is_valid_domain(url_str): - return True - elif is_valid_ipv4(url_str): - return True - elif is_valid_ipv6(url_str): - return True + if url_str: + if is_valid_domain(url_str): + return True + ip_with_port = url_str.split(':') + if len(ip_with_port) <= 2: + # check ipv4 or ipv4 with port + return is_valid_ipv4(ip_with_port[0]) + else: + # check ipv6 with port + if '[' in url_str: + try: + bkt_idx = url_str.index(']') + if bkt_idx + 1 == len(url_str): + # brackets without port + return False + else: + return is_valid_ipv6(url_str[1:bkt_idx]) + except Exception: + return False + else: + # check ipv6 without port + return is_valid_ipv6(url_str) else: return False +def is_valid_bool_str(val): + """Check if the provided string is a valid bool string or not.""" + boolstrs = ('true', 'false') + return str(val).lower() in boolstrs + + def validate_address_str(ip_address_str, network): """Determine whether an address is valid.""" try: diff --git a/configutilities/configutilities/configutilities/common/validator.py b/configutilities/configutilities/configutilities/common/validator.py index 976c132cb1..c635439a83 100644 --- a/configutilities/configutilities/configutilities/common/validator.py +++ b/configutilities/configutilities/configutilities/common/validator.py @@ -26,6 +26,7 @@ from configutilities.common.utils import validate_address_str from configutilities.common.utils import validate_nameserver_address_str from configutilities.common.utils import is_valid_url from configutilities.common.utils import is_valid_domain_or_ip +from configutilities.common.utils import is_valid_bool_str from configutilities.common.exceptions import ConfigFail from configutilities.common.exceptions import ValidateFail @@ -1010,6 +1011,77 @@ class ConfigValidator(object): self.cgcs_conf.set('cDOCKER_PROXY', 'DOCKER_NO_PROXY', docker_no_proxy_list_str) + def validate_docker_registry(self): + if not self.conf.has_section('DOCKER_REGISTRY'): + return + if self.cgcs_conf is not None: + self.cgcs_conf.add_section('cDOCKER_REGISTRY') + # check k8s_registry + if self.conf.has_option('DOCKER_REGISTRY', 'DOCKER_K8S_REGISTRY'): + docker_k8s_registry_str = self.conf.get( + 'DOCKER_REGISTRY', 'DOCKER_K8S_REGISTRY') + if is_valid_domain_or_ip(docker_k8s_registry_str): + if self.cgcs_conf is not None: + self.cgcs_conf.set('cDOCKER_REGISTRY', + 'DOCKER_K8S_REGISTRY', + docker_k8s_registry_str) + else: + raise ConfigFail( + "Invalid DOCKER_K8S_REGISTRY value of %s." % + docker_k8s_registry_str) + # check gcr_registry + if self.conf.has_option('DOCKER_REGISTRY', 'DOCKER_GCR_REGISTRY'): + docker_gcr_registry_str = self.conf.get( + 'DOCKER_REGISTRY', 'DOCKER_GCR_REGISTRY') + if is_valid_domain_or_ip(docker_gcr_registry_str): + if self.cgcs_conf is not None: + self.cgcs_conf.set('cDOCKER_REGISTRY', + 'DOCKER_GCR_REGISTRY', + docker_gcr_registry_str) + else: + raise ConfigFail( + "Invalid DOCKER_GCR_REGISTRY value of %s." % + docker_gcr_registry_str) + # check quay_registry + if self.conf.has_option('DOCKER_REGISTRY', 'DOCKER_QUAY_REGISTRY'): + docker_quay_registry_str = self.conf.get( + 'DOCKER_REGISTRY', 'DOCKER_QUAY_REGISTRY') + if is_valid_domain_or_ip(docker_quay_registry_str): + if self.cgcs_conf is not None: + self.cgcs_conf.set('cDOCKER_REGISTRY', + 'DOCKER_QUAY_REGISTRY', + docker_quay_registry_str) + else: + raise ConfigFail( + "Invalid DOCKER_QUAY_REGISTRY value of %s." % + docker_quay_registry_str) + # check docker_registry + if self.conf.has_option('DOCKER_REGISTRY', 'DOCKER_DOCKER_REGISTRY'): + docker_docker_registry_str = self.conf.get( + 'DOCKER_REGISTRY', 'DOCKER_DOCKER_REGISTRY') + if is_valid_domain_or_ip(docker_docker_registry_str): + if self.cgcs_conf is not None: + self.cgcs_conf.set('cDOCKER_REGISTRY', + 'DOCKER_DOCKER_REGISTRY', + docker_docker_registry_str) + else: + raise ConfigFail( + "Invalid DOCKER_DOCKER_REGISTRY value of %s." % + docker_docker_registry_str) + # check is_secure_registry + if self.conf.has_option('DOCKER_REGISTRY', 'IS_SECURE_REGISTRY'): + docker_is_secure_registry_str = self.conf.get( + 'DOCKER_REGISTRY', 'IS_SECURE_REGISTRY') + if is_valid_bool_str(docker_is_secure_registry_str): + if self.cgcs_conf is not None: + self.cgcs_conf.set('cDOCKER_REGISTRY', + 'IS_SECURE_REGISTRY', + docker_is_secure_registry_str) + else: + raise ConfigFail( + "Invalid IS_SECURE_REGISTRY value of %s." % + docker_is_secure_registry_str) + def validate_ntp(self): if self.conf.has_section('NTP'): raise ConfigFail("NTP Configuration is no longer supported") @@ -1468,6 +1540,8 @@ def validate(system_config, config_type=REGION_CONFIG, cgcs_config=None, validator.validate_dns() # Docker Proxy configuration validator.validate_docker_proxy() + # Docker Registry configuration + validator.validate_docker_registry() # NTP configuration validator.validate_ntp() # Network configuration diff --git a/controllerconfig/controllerconfig/controllerconfig/configassistant.py b/controllerconfig/controllerconfig/controllerconfig/configassistant.py index 2915f6b783..e6dd7f5281 100644 --- a/controllerconfig/controllerconfig/controllerconfig/configassistant.py +++ b/controllerconfig/controllerconfig/controllerconfig/configassistant.py @@ -470,6 +470,14 @@ class ConfigAssistant(): self.docker_https_proxy = "" self.docker_no_proxy = "" + # Docker registry config + self.docker_use_default_registry = True + self.docker_k8s_registry = "" + self.docker_gcr_registry = "" + self.docker_quay_registry = "" + self.docker_docker_registry = "" + self.is_secure_registry = True + # SDN config self.enable_sdn = False @@ -2741,7 +2749,7 @@ class ConfigAssistant(): """Allow user to input docker proxy config.""" print("\nDocker Proxy:") - print("-------------------------\n") + print("-------------\n") print(textwrap.fill( "Docker proxy is needed if host OAM network is behind a proxy.", 80)) @@ -2762,7 +2770,7 @@ class ConfigAssistant(): self.docker_http_proxy = user_input break else: - print("Please input a valid url") + print("Please enter a valid url") continue else: self.docker_http_proxy = "" @@ -2778,7 +2786,7 @@ class ConfigAssistant(): self.docker_https_proxy = user_input break else: - print("Please input a valid url") + print("Please enter a valid url") continue else: self.docker_https_proxy = "" @@ -2849,6 +2857,145 @@ class ConfigAssistant(): print("Invalid choice") continue + def input_docker_registry_config(self): + """Allow user to input docker registry config.""" + + print("\nDocker Registry:") + print("----------------\n") + print("Configure docker registries to pull images from.\n" + "Default registries are:\n" + "k8s.gcr.io, gcr.io, quay.io, docker.io\n" + ) + while True: + user_input = input( + "Use default docker registries [Y/n]: ") + if user_input.lower() == 'q': + raise UserQuit + elif user_input.lower() == 'n': + # unexpected newline displayed if textwrap.fill with + # '\n' included + print("\nEach registry can be specified as one of the" + "following:\n" + " - domain (e.g. example.domain)\n" + " - domain with port (e.g. example.domain:5000)\n" + " - IPv4 address (e.g. 1.2.3.4)\n" + " - IPv4 address with port (e.g. 1.2.3.4:5000)\n" + ) + while True: + user_input = input( + "Use a unified registry replacing all " + "default registries [y/n]: ") + if user_input.lower() == 'q': + raise UserQuit + elif user_input.lower() == 'y': + # Input a unified registry to avoid + # inputing the same registry repeatly + while True: + user_input = input( + "Enter a unified registry: ") + if user_input.lower() == 'q': + raise UserQuit + if is_valid_domain_or_ip(user_input): + self.docker_k8s_registry = user_input + self.docker_gcr_registry = user_input + self.docker_quay_registry = user_input + self.docker_docker_registry = user_input + self.docker_use_default_registry = False + break + else: + print("Please enter a valid registry address") + continue + + # Only if a unified registry set, it could be + # an insecure registry + while True: + user_input = input( + "Is '" + self.docker_k8s_registry + + "' a secure registry (https) [Y/n]: ") + if user_input.lower() == 'q': + raise UserQuit + elif user_input.lower() in ('y', ''): + self.is_secure_registry = True + break + elif user_input.lower() == 'n': + self.is_secure_registry = False + break + else: + print("Invalid choice") + continue + break + + elif user_input.lower() == 'n': + # Input alternative registries separately + while True: + user_input = input( + "Alternative registry to k8s.gcr.io: ") + if user_input.lower() == 'q': + raise UserQuit + if is_valid_domain_or_ip(user_input): + self.docker_k8s_registry = user_input + break + else: + print("Please enter a valid registry address") + continue + + while True: + user_input = input( + "Alternative registry to gcr.io: ") + if user_input.lower() == 'q': + raise UserQuit + if is_valid_domain_or_ip(user_input): + self.docker_gcr_registry = user_input + break + else: + print("Please enter a valid registry address") + continue + + while True: + user_input = input( + "Alternative registry to quay.io: ") + if user_input.lower() == 'q': + raise UserQuit + if is_valid_domain_or_ip(user_input): + self.docker_quay_registry = user_input + break + else: + print("Please enter a valid registry address") + continue + + while True: + user_input = input( + "Alternative registry to docker.io: ") + if user_input.lower() == 'q': + raise UserQuit + if is_valid_domain_or_ip(user_input): + self.docker_docker_registry = user_input + break + else: + print("Please enter a valid registry address") + continue + + if (self.docker_k8s_registry or + self.docker_gcr_registry or + self.docker_quay_registry or + self.docker_docker_registry): + self.docker_use_default_registry = False + break + else: + print("At least one registry is required") + continue + else: + print("Invalid choice") + continue + break + + elif user_input.lower() in ('y', ''): + self.docker_use_default_registry = True + break + else: + print("Invalid choice") + continue + def input_authentication_config(self): """Allow user to input authentication config and perform validation. """ @@ -2936,6 +3083,7 @@ class ConfigAssistant(): self.input_dns_config() # Docker proxy is only used in kubernetes config self.input_docker_proxy_config() + self.input_docker_registry_config() self.input_authentication_config() def is_valid_management_multicast_subnet(self, ip_subnet): @@ -3298,6 +3446,32 @@ class ConfigAssistant(): self.docker_no_proxy = config.get( 'cDOCKER_PROXY', 'DOCKER_NO_PROXY') + # Docker Registry Configuration + if config.has_section('cDOCKER_REGISTRY'): + self.docker_use_default_registry = False + if config.has_option('cDOCKER_REGISTRY', + 'DOCKER_K8S_REGISTRY'): + self.docker_k8s_registry = config.get( + 'cDOCKER_REGISTRY', 'DOCKER_K8S_REGISTRY') + if config.has_option('cDOCKER_REGISTRY', + 'DOCKER_GCR_REGISTRY'): + self.docker_gcr_registry = config.get( + 'cDOCKER_REGISTRY', 'DOCKER_GCR_REGISTRY') + if config.has_option('cDOCKER_REGISTRY', + 'DOCKER_QUAY_REGISTRY'): + self.docker_quay_registry = config.get( + 'cDOCKER_REGISTRY', 'DOCKER_QUAY_REGISTRY') + if config.has_option('cDOCKER_REGISTRY', + 'DOCKER_DOCKER_REGISTRY'): + self.docker_docker_registry = config.get( + 'cDOCKER_REGISTRY', 'DOCKER_DOCKER_REGISTRY') + if config.has_option('cDOCKER_REGISTRY', + 'IS_SECURE_REGISTRY'): + self.is_secure_registry = config.getboolean( + 'cDOCKER_REGISTRY', 'IS_SECURE_REGISTRY') + else: + self.is_secure_registry = True + # SDN Network configuration if config.has_option('cSDN', 'ENABLE_SDN'): raise ConfigFail("The option ENABLE_SDN is no longer " @@ -3742,14 +3916,31 @@ class ConfigAssistant(): if not dns_config: print("External DNS servers not configured") if self.enable_docker_proxy: - print("\nDocker Proxy Configuraton") - print("----------------------") + print("\nDocker Proxy Configuration") + print("--------------------------") if self.docker_http_proxy: print("Docker HTTP proxy: " + self.docker_http_proxy) if self.docker_https_proxy: print("Docker HTTPS proxy: " + self.docker_https_proxy) if self.docker_no_proxy: print("Docker NO proxy: " + self.docker_no_proxy) + if not self.docker_use_default_registry: + print("\nDocker Registry Configuration") + print("-----------------------------") + if self.docker_k8s_registry: + print("Alternative registry to k8s.gcr.io: " + + self.docker_k8s_registry) + if self.docker_gcr_registry: + print("Alternative registry to gcr.io: " + + self.docker_gcr_registry) + if self.docker_quay_registry: + print("Alternative registry to quay.io: " + + self.docker_quay_registry) + if self.docker_docker_registry: + print("Alternative registry to docker.io: " + + self.docker_docker_registry) + print("Is registries secure: " + + str(self.is_secure_registry)) if self.region_config: print("\nRegion Configuration") @@ -4064,6 +4255,30 @@ class ConfigAssistant(): "DOCKER_NO_PROXY=" + str(self.docker_no_proxy) + "\n") + # Docker registry configuration + if not self.docker_use_default_registry: + f.write("\n[cDOCKER_REGISTRY]") + f.write("\n# Docker Registry Configuration\n") + if self.docker_k8s_registry: + f.write( + "DOCKER_K8S_REGISTRY=" + + str(self.docker_k8s_registry) + "\n") + if self.docker_gcr_registry: + f.write( + "DOCKER_GCR_REGISTRY=" + + str(self.docker_gcr_registry) + "\n") + if self.docker_quay_registry: + f.write( + "DOCKER_QUAY_REGISTRY=" + + str(self.docker_quay_registry) + "\n") + if self.docker_docker_registry: + f.write( + "DOCKER_DOCKER_REGISTRY=" + + str(self.docker_docker_registry) + "\n") + f.write( + "IS_SECURE_REGISTRY=" + + str(self.is_secure_registry) + "\n") + # Network configuration f.write("\n[cNETWORK]") f.write("\n# Data Network Configuration\n") @@ -5468,22 +5683,54 @@ class ConfigAssistant(): client.sysinv.idns.update(dns_record.uuid, patch) def _populate_docker_config(self, client): - parameter = {} - if self.docker_http_proxy: - parameter['http_proxy'] = self.docker_http_proxy - if self.docker_https_proxy: - parameter['https_proxy'] = self.docker_https_proxy - if self.docker_no_proxy: - parameter['no_proxy'] = self.docker_no_proxy + if self.enable_docker_proxy: + proxy_parameter = {} + if self.docker_http_proxy: + proxy_parameter['http_proxy'] = self.docker_http_proxy + if self.docker_https_proxy: + proxy_parameter['https_proxy'] = self.docker_https_proxy + if self.docker_no_proxy: + proxy_parameter['no_proxy'] = self.docker_no_proxy - if parameter: - client.sysinv.service_parameter.create( - sysinv_constants.SERVICE_TYPE_DOCKER, - sysinv_constants.SERVICE_PARAM_SECTION_DOCKER_PROXY, - None, - None, - parameter - ) + if proxy_parameter: + client.sysinv.service_parameter.create( + sysinv_constants.SERVICE_TYPE_DOCKER, + sysinv_constants.SERVICE_PARAM_SECTION_DOCKER_PROXY, + None, + None, + proxy_parameter + ) + + if not self.docker_use_default_registry: + registry_parameter = {} + if self.docker_k8s_registry: + registry_parameter['k8s'] = \ + self.docker_k8s_registry + + if self.docker_gcr_registry: + registry_parameter['gcr'] = \ + self.docker_gcr_registry + + if self.docker_quay_registry: + registry_parameter['quay'] = \ + self.docker_quay_registry + + if self.docker_docker_registry: + registry_parameter['docker'] = \ + self.docker_docker_registry + + if not self.is_secure_registry: + registry_parameter['insecure_registry'] = "True" + + if registry_parameter: + client.sysinv.service_parameter.create( + sysinv_constants.SERVICE_TYPE_DOCKER, + sysinv_constants. + SERVICE_PARAM_SECTION_DOCKER_REGISTRY, + None, + None, + registry_parameter + ) def populate_initial_config(self): """Populate initial system inventory configuration""" @@ -5494,8 +5741,7 @@ class ConfigAssistant(): self._populate_network_config(client) if self.kubernetes: self._populate_dns_config(client) - if self.enable_docker_proxy: - self._populate_docker_config(client) + self._populate_docker_config(client) controller = self._populate_controller_config(client) # ceph_mon config requires controller host to be created self._inventory_config_complete_wait(client, controller) diff --git a/controllerconfig/controllerconfig/controllerconfig/tests/files/cgcs_config.kubernetes b/controllerconfig/controllerconfig/controllerconfig/tests/files/cgcs_config.kubernetes index 7df235ba86..2f959d5d5e 100755 --- a/controllerconfig/controllerconfig/controllerconfig/tests/files/cgcs_config.kubernetes +++ b/controllerconfig/controllerconfig/controllerconfig/tests/files/cgcs_config.kubernetes @@ -79,6 +79,14 @@ DOCKER_HTTP_PROXY=http://proxy.com:123 DOCKER_HTTPS_PROXY=https://proxy.com:123 DOCKER_NO_PROXY=localhost,127.0.0.1,192.168.204.2 +[cDOCKER_REGISTRY] +# Docker Registry Configuration +DOCKER_K8S_REGISTRY=my.registry.com:5000 +DOCKER_GCR_REGISTRY=my.registry.com +DOCKER_QUAY_REGISTRY=1.2.3.4:5000 +DOCKER_DOCKER_REGISTRY=[1:2:3:4:a:b:c:d]:5000 +IS_SECURE_REGISTRY=False + [cNETWORK] # Data Network Configuration VSWITCH_TYPE=ovs-dpdk diff --git a/controllerconfig/controllerconfig/controllerconfig/tests/files/system_config.kubernetes b/controllerconfig/controllerconfig/controllerconfig/tests/files/system_config.kubernetes index 7ff90974ac..0a1d209714 100755 --- a/controllerconfig/controllerconfig/controllerconfig/tests/files/system_config.kubernetes +++ b/controllerconfig/controllerconfig/controllerconfig/tests/files/system_config.kubernetes @@ -55,6 +55,14 @@ DOCKER_HTTP_PROXY=http://proxy.com:123 DOCKER_HTTPS_PROXY=https://proxy.com:123 DOCKER_NO_PROXY=localhost,127.0.0.1,192.168.204.2 +[DOCKER_REGISTRY] +# Docker Registry Configuration +DOCKER_K8S_REGISTRY=my.registry.com:5000 +DOCKER_GCR_REGISTRY=my.registry.com +DOCKER_QUAY_REGISTRY=1.2.3.4:5000 +DOCKER_DOCKER_REGISTRY=[1:2:3:4:a:b:c:d]:5000 +IS_SECURE_REGISTRY=False + ;[PXEBOOT_NETWORK] ;PXEBOOT_CIDR=192.168.203.0/24 diff --git a/controllerconfig/controllerconfig/controllerconfig/tests/test_system_config.py b/controllerconfig/controllerconfig/controllerconfig/tests/test_system_config.py index a8438eb8ee..2bda2825c9 100644 --- a/controllerconfig/controllerconfig/controllerconfig/tests/test_system_config.py +++ b/controllerconfig/controllerconfig/controllerconfig/tests/test_system_config.py @@ -654,3 +654,10 @@ def test_kubernetes(): cr.create_cgcs_config_file(None, system_config, None, None, None, 0, validate_only=True) validate(system_config, DEFAULT_CONFIG, None, False) + + # Test absence of optional docker registry configuration + system_config = cr.parse_system_config(systemfile) + system_config.remove_section('DOCKER_REGISTRY') + cr.create_cgcs_config_file(None, system_config, None, None, None, 0, + validate_only=True) + validate(system_config, DEFAULT_CONFIG, None, False) diff --git a/puppet-manifests/src/modules/platform/manifests/docker.pp b/puppet-manifests/src/modules/platform/manifests/docker.pp index 23beb2e884..2e67dbf113 100644 --- a/puppet-manifests/src/modules/platform/manifests/docker.pp +++ b/puppet-manifests/src/modules/platform/manifests/docker.pp @@ -3,6 +3,11 @@ class platform::docker::params ( $http_proxy = undef, $https_proxy = undef, $no_proxy = undef, + $k8s_registry = undef, + $gcr_registry = undef, + $quay_registry = undef, + $docker_registry = undef, + $insecure_registry = undef, ) { } class platform::docker::config diff --git a/puppet-manifests/src/modules/platform/manifests/dockerdistribution.pp b/puppet-manifests/src/modules/platform/manifests/dockerdistribution.pp index 6aaecee5bd..453ab7dadd 100644 --- a/puppet-manifests/src/modules/platform/manifests/dockerdistribution.pp +++ b/puppet-manifests/src/modules/platform/manifests/dockerdistribution.pp @@ -7,9 +7,18 @@ class platform::dockerdistribution::config if $enabled { include ::platform::network::mgmt::params + include ::platform::docker::params $docker_registry_ip = $::platform::network::mgmt::params::controller_address + # check insecure registries + if $::platform::docker::params::insecure_registry { + # insecure registry is true means unified registry was set + $insecure_registries = "\"${::platform::docker::params::k8s_registry}\", \"${docker_registry_ip}:9001\"" + } else { + $insecure_registries = "\"${docker_registry_ip}:9001\"" + } + # currently docker registry is running insecure mode # when proper authentication is implemented, this would go away file { '/etc/docker': @@ -52,9 +61,18 @@ class platform::dockerdistribution::compute $enabled = $::platform::kubernetes::params::enabled if $enabled { include ::platform::network::mgmt::params + include ::platform::docker::params $docker_registry_ip = $::platform::network::mgmt::params::controller_address + # check insecure registries + if $::platform::docker::params::insecure_registry { + # insecure registry is true means unified registry was set + $insecure_registries = "\"${::platform::docker::params::k8s_registry}\", \"${docker_registry_ip}:9001\"" + } else { + $insecure_registries = "\"${docker_registry_ip}:9001\"" + } + # currently docker registry is running insecure mode # when proper authentication is implemented, this would go away file { '/etc/docker': diff --git a/puppet-manifests/src/modules/platform/manifests/helm.pp b/puppet-manifests/src/modules/platform/manifests/helm.pp index 31396dae90..f92c54c487 100644 --- a/puppet-manifests/src/modules/platform/manifests/helm.pp +++ b/puppet-manifests/src/modules/platform/manifests/helm.pp @@ -7,6 +7,7 @@ class platform::helm inherits ::platform::helm::repository::params { include ::platform::kubernetes::params + include ::platform::docker::params if $::platform::kubernetes::params::enabled { file {$source_helm_repo_dir: @@ -27,17 +28,29 @@ class platform::helm if str2bool($::is_initial_config_primary) { + if $::platform::docker::params::gcr_registry { + $gcr_registry = $::platform::docker::params::gcr_registry + } else { + $gcr_registry = 'gcr.io' + } + + if $::platform::docker::params::quay_registry { + $quay_registry = $::platform::docker::params::quay_registry + } else { + $quay_registry = 'quay.io' + } + Class['::platform::kubernetes::master'] # TODO(jrichard): Upversion tiller image to v2.11.1 once released. -> exec { 'load tiller docker image': - command => 'docker image pull gcr.io/kubernetes-helm/tiller:v2.12.1', + command => "docker image pull ${gcr_registry}/kubernetes-helm/tiller:v2.12.1", logoutput => true, } # TODO(tngo): If and when tiller image is upversioned, please ensure armada compatibility as part of the test -> exec { 'load armada docker image': - command => 'docker image pull quay.io/airshipit/armada:f807c3a1ec727c883c772ffc618f084d960ed5c9', + command => "docker image pull ${quay_registry}/airshipit/armada:f807c3a1ec727c883c772ffc618f084d960ed5c9", logoutput => true, } @@ -54,7 +67,7 @@ class platform::helm # TODO(jrichard): Upversion tiller image to v2.11.1 once released. -> exec { 'initialize helm': environment => [ 'KUBECONFIG=/etc/kubernetes/admin.conf', 'HOME=/home/wrsroot' ], - command => 'helm init --skip-refresh --service-account tiller --node-selectors "node-role.kubernetes.io/master"="" --tiller-image=gcr.io/kubernetes-helm/tiller:v2.12.1', # lint:ignore:140chars + command => "helm init --skip-refresh --service-account tiller --node-selectors \"node-role.kubernetes.io/master\"=\"\" --tiller-image=${gcr_registry}/kubernetes-helm/tiller:v2.12.1", # lint:ignore:140chars logoutput => true, user => 'wrsroot', group => 'wrs', diff --git a/puppet-manifests/src/modules/platform/manifests/kubernetes.pp b/puppet-manifests/src/modules/platform/manifests/kubernetes.pp index 49c28fdba9..e68b852b44 100644 --- a/puppet-manifests/src/modules/platform/manifests/kubernetes.pp +++ b/puppet-manifests/src/modules/platform/manifests/kubernetes.pp @@ -13,9 +13,27 @@ class platform::kubernetes::params ( ) { } class platform::kubernetes::kubeadm { + include ::platform::docker::params + $iptables_file = "net.bridge.bridge-nf-call-ip6tables = 1 net.bridge.bridge-nf-call-iptables = 1" + if $::platform::docker::params::k8s_registry { + $k8s_registry = $::platform::docker::params::k8s_registry + } else { + $k8s_registry = undef + } + + # kubelet use --pod-infra-container-image to indentify the specified image + # TODO: this is not needed after kubernetes upgraded to 1.13 + # because the imageRepository setting will be used + if $k8s_registry { + file { '/etc/sysconfig/kubelet': + ensure => file, + content => template('platform/kubelet.conf.erb'), + } + } + # Update iptables config. This is required based on: # https://kubernetes.io/docs/tasks/tools/install-kubeadm # This probably belongs somewhere else - initscripts package? @@ -52,6 +70,21 @@ class platform::kubernetes::master::init inherits ::platform::kubernetes::params { include ::platform::params + include ::platform::docker::params + + # This is used for imageRepository in template kubeadm.yaml.erb + if $::platform::docker::params::k8s_registry { + $k8s_registry = $::platform::docker::params::k8s_registry + } else { + $k8s_registry = undef + } + + # This is used for calico image in template calico.yaml.erb + if $::platform::docker::params::quay_registry { + $quay_registry = $::platform::docker::params::quay_registry + } else { + $quay_registry = 'quay.io' + } if str2bool($::is_initial_config_primary) { # For initial controller install, configure kubernetes from scratch. @@ -347,10 +380,11 @@ class platform::kubernetes::worker } # TODO: remove port 9001 once we have a public docker image registry using standard ports. +# add 5000 as the default port for private registry class platform::kubernetes::firewall::params ( $transport = 'tcp', $table = 'nat', - $dports = [80, 443, 9001], + $dports = [80, 443, 9001, 5000], $chain = 'POSTROUTING', $jump = 'SNAT', ) {} diff --git a/puppet-manifests/src/modules/platform/templates/calico.yaml.erb b/puppet-manifests/src/modules/platform/templates/calico.yaml.erb index 1f6eea5ffc..6c6a439cf6 100644 --- a/puppet-manifests/src/modules/platform/templates/calico.yaml.erb +++ b/puppet-manifests/src/modules/platform/templates/calico.yaml.erb @@ -107,7 +107,7 @@ spec: # as a host-networked pod. serviceAccountName: calico-node containers: - - image: quay.io/calico/typha:v3.1.4 + - image: <%= @quay_registry %>/calico/typha:v3.1.4 name: calico-typha ports: - containerPort: 5473 @@ -198,7 +198,7 @@ spec: # container programs network policy and routes on each # host. - name: calico-node - image: quay.io/calico/node:v3.1.4 + image: <%= @quay_registry %>/calico/node:v3.1.4 env: # Use Kubernetes API as the backing datastore. - name: DATASTORE_TYPE @@ -283,7 +283,7 @@ spec: # This container installs the Calico CNI binaries # and CNI network config file on each node. - name: install-cni - image: quay.io/calico/cni:v3.1.4 + image: <%= @quay_registry %>/calico/cni:v3.1.4 command: ["/install-cni.sh"] env: # Name of the CNI config file to create. diff --git a/puppet-manifests/src/modules/platform/templates/insecuredockerregistry.conf.erb b/puppet-manifests/src/modules/platform/templates/insecuredockerregistry.conf.erb index 0998bc827f..911fc31300 100644 --- a/puppet-manifests/src/modules/platform/templates/insecuredockerregistry.conf.erb +++ b/puppet-manifests/src/modules/platform/templates/insecuredockerregistry.conf.erb @@ -1,3 +1,3 @@ { - "insecure-registries" : [ "<%= @docker_registry_ip %>:9001" ] + "insecure-registries" : [ <%= @insecure_registries %> ] } diff --git a/puppet-manifests/src/modules/platform/templates/kubeadm.yaml.erb b/puppet-manifests/src/modules/platform/templates/kubeadm.yaml.erb index c64b03471b..32efb4b615 100644 --- a/puppet-manifests/src/modules/platform/templates/kubeadm.yaml.erb +++ b/puppet-manifests/src/modules/platform/templates/kubeadm.yaml.erb @@ -21,6 +21,9 @@ controllerManagerExtraArgs: node-monitor-period: "2s" node-monitor-grace-period: "20s" pod-eviction-timeout: "30s" +<%- if @k8s_registry -%> +imageRepository: "<%= @k8s_registry %>" +<%- end -%> --- kind: KubeletConfiguration apiVersion: kubelet.config.k8s.io/v1beta1 diff --git a/puppet-manifests/src/modules/platform/templates/kubelet.conf.erb b/puppet-manifests/src/modules/platform/templates/kubelet.conf.erb new file mode 100644 index 0000000000..6c36049dfd --- /dev/null +++ b/puppet-manifests/src/modules/platform/templates/kubelet.conf.erb @@ -0,0 +1,2 @@ +# Overrides config file for kubelet +KUBELET_EXTRA_ARGS=--pod-infra-container-image=<%= @k8s_registry %>/pause:3.1 diff --git a/puppet-modules-wrs/puppet-sysinv/src/sysinv/manifests/init.pp b/puppet-modules-wrs/puppet-sysinv/src/sysinv/manifests/init.pp index 2fa57491e6..9dd51a8add 100644 --- a/puppet-modules-wrs/puppet-sysinv/src/sysinv/manifests/init.pp +++ b/puppet-modules-wrs/puppet-sysinv/src/sysinv/manifests/init.pp @@ -77,6 +77,7 @@ class sysinv ( include sysinv::params include ::platform::kubernetes::params + include ::platform::docker::params Package['sysinv'] -> Sysinv_config<||> Package['sysinv'] -> Sysinv_api_paste_ini<||> @@ -217,7 +218,13 @@ class sysinv ( } if $::platform::kubernetes::params::enabled == true { - $armada_img_tag = 'quay.io/airshipit/armada:f807c3a1ec727c883c772ffc618f084d960ed5c9' + if $::platform::docker::params::quay_registry { + $quay_registry = $::platform::docker::params::quay_registry + } else { + $quay_registry = 'quay.io' + } + + $armada_img_tag = "${quay_registry}/airshipit/armada:f807c3a1ec727c883c772ffc618f084d960ed5c9" sysinv_config { 'DEFAULT/armada_image_tag': value => $armada_img_tag; } diff --git a/sysinv/sysinv/sysinv/sysinv/common/constants.py b/sysinv/sysinv/sysinv/sysinv/common/constants.py index 682612e29d..132dbdfa9a 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/constants.py +++ b/sysinv/sysinv/sysinv/sysinv/common/constants.py @@ -1103,6 +1103,12 @@ SERVICE_PARAM_SECTION_DOCKER_PROXY = 'proxy' SERVICE_PARAM_NAME_DOCKER_HTTP_PROXY = 'http_proxy' SERVICE_PARAM_NAME_DOCKER_HTTPS_PROXY = 'https_proxy' SERVICE_PARAM_NAME_DOCKER_NO_PROXY = 'no_proxy' +SERVICE_PARAM_SECTION_DOCKER_REGISTRY = 'registry' +SERVICE_PARAM_NAME_DOCKER_K8S_REGISTRY = 'k8s' +SERVICE_PARAM_NAME_DOCKER_GCR_REGISTRY = 'gcr' +SERVICE_PARAM_NAME_DOCKER_QUAY_REGISTRY = 'quay' +SERVICE_PARAM_NAME_DOCKER_DOCKER_REGISTRY = 'docker' +SERVICE_PARAM_NAME_DOCKER_INSECURE_REGISTRY = 'insecure_registry' # default filesystem size to 25 MB SERVICE_PARAM_SWIFT_FS_SIZE_MB_DEFAULT = 25 diff --git a/sysinv/sysinv/sysinv/sysinv/common/service_parameter.py b/sysinv/sysinv/sysinv/sysinv/common/service_parameter.py index 1dac250187..8d7c014a05 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/service_parameter.py +++ b/sysinv/sysinv/sysinv/sysinv/common/service_parameter.py @@ -660,11 +660,24 @@ def _validate_docker_no_proxy_address(name, value): values = value.split(',') for item in values: # will extend to more cases if CIDR notation is supported - if not cutils.is_valid_domain(item): - if not cutils.is_valid_ip(item): - raise wsme.exc.ClientSideError(_( - "Parameter '%s' includes an invalid address '%s'." % - (name, item))) + if not cutils.is_valid_domain_or_ip(item): + raise wsme.exc.ClientSideError(_( + "Parameter '%s' includes an invalid address '%s'." % + (name, item))) + + +def _validate_docker_registry_address(name, value): + """Check if registry address is valid""" + if not cutils.is_valid_domain_or_ip(value): + raise wsme.exc.ClientSideError(_( + "Parameter '%s' must be a valid address." % name)) + + +def _validate_docker_insecure_registry_bool(name, value): + """Check if insecure registry is a valid bool""" + if not cutils.is_valid_boolstr(value): + raise wsme.exc.ClientSideError(_( + "Parameter '%s' must be a valid bool string." % name)) # LDAP Identity Service Parameters (mandatory) @@ -1516,6 +1529,35 @@ DOCKER_PROXY_PARAMETER_RESOURCE = { 'platform::docker::params::no_proxy', } +DOCKER_REGISTRY_PARAMETER_OPTIONAL = [ + constants.SERVICE_PARAM_NAME_DOCKER_K8S_REGISTRY, + constants.SERVICE_PARAM_NAME_DOCKER_GCR_REGISTRY, + constants.SERVICE_PARAM_NAME_DOCKER_QUAY_REGISTRY, + constants.SERVICE_PARAM_NAME_DOCKER_DOCKER_REGISTRY, + constants.SERVICE_PARAM_NAME_DOCKER_INSECURE_REGISTRY, +] + +DOCKER_REGISTRY_PARAMETER_VALIDATOR = { + constants.SERVICE_PARAM_NAME_DOCKER_K8S_REGISTRY: _validate_docker_registry_address, + constants.SERVICE_PARAM_NAME_DOCKER_GCR_REGISTRY: _validate_docker_registry_address, + constants.SERVICE_PARAM_NAME_DOCKER_QUAY_REGISTRY: _validate_docker_registry_address, + constants.SERVICE_PARAM_NAME_DOCKER_DOCKER_REGISTRY: _validate_docker_registry_address, + constants.SERVICE_PARAM_NAME_DOCKER_INSECURE_REGISTRY: _validate_docker_insecure_registry_bool, +} + +DOCKER_REGISTRY_PARAMETER_RESOURCE = { + constants.SERVICE_PARAM_NAME_DOCKER_K8S_REGISTRY: + 'platform::docker::params::k8s_registry', + constants.SERVICE_PARAM_NAME_DOCKER_GCR_REGISTRY: + 'platform::docker::params::gcr_registry', + constants.SERVICE_PARAM_NAME_DOCKER_QUAY_REGISTRY: + 'platform::docker::params::quay_registry', + constants.SERVICE_PARAM_NAME_DOCKER_DOCKER_REGISTRY: + 'platform::docker::params::docker_registry', + constants.SERVICE_PARAM_NAME_DOCKER_INSECURE_REGISTRY: + 'platform::docker::params::insecure_registry', +} + HTTPD_PORT_PARAMETER_OPTIONAL = [ constants.SERVICE_PARAM_HTTP_PORT_HTTP, constants.SERVICE_PARAM_HTTP_PORT_HTTPS, @@ -1715,6 +1757,11 @@ SERVICE_PARAMETER_SCHEMA = { SERVICE_PARAM_VALIDATOR: DOCKER_PROXY_PARAMETER_VALIDATOR, SERVICE_PARAM_RESOURCE: DOCKER_PROXY_PARAMETER_RESOURCE, }, + constants.SERVICE_PARAM_SECTION_DOCKER_REGISTRY: { + SERVICE_PARAM_OPTIONAL: DOCKER_REGISTRY_PARAMETER_OPTIONAL, + SERVICE_PARAM_VALIDATOR: DOCKER_REGISTRY_PARAMETER_VALIDATOR, + SERVICE_PARAM_RESOURCE: DOCKER_REGISTRY_PARAMETER_RESOURCE, + }, }, constants.SERVICE_TYPE_HTTP: { constants.SERVICE_PARAM_SECTION_HTTP_CONFIG: { diff --git a/sysinv/sysinv/sysinv/sysinv/common/utils.py b/sysinv/sysinv/sysinv/sysinv/common/utils.py index a87ebc65d4..12d2be41c3 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/utils.py +++ b/sysinv/sysinv/sysinv/sysinv/common/utils.py @@ -1832,6 +1832,33 @@ def is_valid_domain(url_str): return False +def is_valid_domain_or_ip(url_str): + if url_str: + if is_valid_domain(url_str): + return True + ip_with_port = url_str.split(':') + if len(ip_with_port) <= 2: + # check ipv4 or ipv4 with port + return is_valid_ipv4(ip_with_port[0]) + else: + # check ipv6 with port + if '[' in url_str: + try: + bkt_idx = url_str.index(']') + if bkt_idx + 1 == len(url_str): + # brackets without port + return False + else: + return is_valid_ipv6(url_str[1:bkt_idx]) + except Exception: + return False + else: + # check ipv6 without port + return is_valid_ipv6(url_str) + else: + return False + + def verify_checksum(path): """ Find and validate the checksum file in a given directory. """ rc = True