Merge "Allow user specified registries for config_controller" into f/stein

configutilities/configutilities/configutilities/common/utils.py

@@ -182,16 +182,38 @@ def is_valid_ipv6(address):
 
 
 def is_valid_domain_or_ip(url_str):
-    if is_valid_domain(url_str):
+    if url_str:
-        return True
+        if is_valid_domain(url_str):
-    elif is_valid_ipv4(url_str):
+            return True
-        return True
+        ip_with_port = url_str.split(':')
-    elif is_valid_ipv6(url_str):
+        if len(ip_with_port) <= 2:
-        return True
+            # check ipv4 or ipv4 with port
+            return is_valid_ipv4(ip_with_port[0])
+        else:
+            # check ipv6 with port
+            if '[' in url_str:
+                try:
+                    bkt_idx = url_str.index(']')
+                    if bkt_idx + 1 == len(url_str):
+                        # brackets without port
+                        return False
+                    else:
+                        return is_valid_ipv6(url_str[1:bkt_idx])
+                except Exception:
+                    return False
+            else:
+                # check ipv6 without port
+                return is_valid_ipv6(url_str)
     else:
         return False
 
 
+def is_valid_bool_str(val):
+    """Check if the provided string is a valid bool string or not."""
+    boolstrs = ('true', 'false')
+    return str(val).lower() in boolstrs
+
+
 def validate_address_str(ip_address_str, network):
     """Determine whether an address is valid."""
     try:

configutilities/configutilities/configutilities/common/validator.py

@@ -26,6 +26,7 @@ from configutilities.common.utils import validate_address_str
 from configutilities.common.utils import validate_nameserver_address_str
 from configutilities.common.utils import is_valid_url
 from configutilities.common.utils import is_valid_domain_or_ip
+from configutilities.common.utils import is_valid_bool_str
 from configutilities.common.exceptions import ConfigFail
 from configutilities.common.exceptions import ValidateFail
 
@@ -1010,6 +1011,77 @@ class ConfigValidator(object):
                 self.cgcs_conf.set('cDOCKER_PROXY', 'DOCKER_NO_PROXY',
                                    docker_no_proxy_list_str)
 
+    def validate_docker_registry(self):
+        if not self.conf.has_section('DOCKER_REGISTRY'):
+            return
+        if self.cgcs_conf is not None:
+            self.cgcs_conf.add_section('cDOCKER_REGISTRY')
+        # check k8s_registry
+        if self.conf.has_option('DOCKER_REGISTRY', 'DOCKER_K8S_REGISTRY'):
+            docker_k8s_registry_str = self.conf.get(
+                'DOCKER_REGISTRY', 'DOCKER_K8S_REGISTRY')
+            if is_valid_domain_or_ip(docker_k8s_registry_str):
+                if self.cgcs_conf is not None:
+                    self.cgcs_conf.set('cDOCKER_REGISTRY',
+                                       'DOCKER_K8S_REGISTRY',
+                                       docker_k8s_registry_str)
+            else:
+                raise ConfigFail(
+                    "Invalid DOCKER_K8S_REGISTRY value of %s." %
+                    docker_k8s_registry_str)
+        # check gcr_registry
+        if self.conf.has_option('DOCKER_REGISTRY', 'DOCKER_GCR_REGISTRY'):
+            docker_gcr_registry_str = self.conf.get(
+                'DOCKER_REGISTRY', 'DOCKER_GCR_REGISTRY')
+            if is_valid_domain_or_ip(docker_gcr_registry_str):
+                if self.cgcs_conf is not None:
+                    self.cgcs_conf.set('cDOCKER_REGISTRY',
+                                       'DOCKER_GCR_REGISTRY',
+                                       docker_gcr_registry_str)
+            else:
+                raise ConfigFail(
+                    "Invalid DOCKER_GCR_REGISTRY value of %s." %
+                    docker_gcr_registry_str)
+        # check quay_registry
+        if self.conf.has_option('DOCKER_REGISTRY', 'DOCKER_QUAY_REGISTRY'):
+            docker_quay_registry_str = self.conf.get(
+                'DOCKER_REGISTRY', 'DOCKER_QUAY_REGISTRY')
+            if is_valid_domain_or_ip(docker_quay_registry_str):
+                if self.cgcs_conf is not None:
+                    self.cgcs_conf.set('cDOCKER_REGISTRY',
+                                       'DOCKER_QUAY_REGISTRY',
+                                       docker_quay_registry_str)
+            else:
+                raise ConfigFail(
+                    "Invalid DOCKER_QUAY_REGISTRY value of %s." %
+                    docker_quay_registry_str)
+        # check docker_registry
+        if self.conf.has_option('DOCKER_REGISTRY', 'DOCKER_DOCKER_REGISTRY'):
+            docker_docker_registry_str = self.conf.get(
+                'DOCKER_REGISTRY', 'DOCKER_DOCKER_REGISTRY')
+            if is_valid_domain_or_ip(docker_docker_registry_str):
+                if self.cgcs_conf is not None:
+                    self.cgcs_conf.set('cDOCKER_REGISTRY',
+                                       'DOCKER_DOCKER_REGISTRY',
+                                       docker_docker_registry_str)
+            else:
+                raise ConfigFail(
+                    "Invalid DOCKER_DOCKER_REGISTRY value of %s." %
+                    docker_docker_registry_str)
+        # check is_secure_registry
+        if self.conf.has_option('DOCKER_REGISTRY', 'IS_SECURE_REGISTRY'):
+            docker_is_secure_registry_str = self.conf.get(
+                'DOCKER_REGISTRY', 'IS_SECURE_REGISTRY')
+            if is_valid_bool_str(docker_is_secure_registry_str):
+                if self.cgcs_conf is not None:
+                    self.cgcs_conf.set('cDOCKER_REGISTRY',
+                                       'IS_SECURE_REGISTRY',
+                                       docker_is_secure_registry_str)
+            else:
+                raise ConfigFail(
+                    "Invalid IS_SECURE_REGISTRY value of %s." %
+                    docker_is_secure_registry_str)
+
     def validate_ntp(self):
         if self.conf.has_section('NTP'):
             raise ConfigFail("NTP Configuration is no longer supported")
@@ -1468,6 +1540,8 @@ def validate(system_config, config_type=REGION_CONFIG, cgcs_config=None,
     validator.validate_dns()
     # Docker Proxy configuration
     validator.validate_docker_proxy()
+    # Docker Registry configuration
+    validator.validate_docker_registry()
     # NTP configuration
     validator.validate_ntp()
     # Network configuration

controllerconfig/controllerconfig/controllerconfig/configassistant.py

@@ -470,6 +470,14 @@ class ConfigAssistant():
         self.docker_https_proxy = ""
         self.docker_no_proxy = ""
 
+        # Docker registry config
+        self.docker_use_default_registry = True
+        self.docker_k8s_registry = ""
+        self.docker_gcr_registry = ""
+        self.docker_quay_registry = ""
+        self.docker_docker_registry = ""
+        self.is_secure_registry = True
+
         # SDN config
         self.enable_sdn = False
 
@@ -2741,7 +2749,7 @@ class ConfigAssistant():
         """Allow user to input docker proxy config."""
 
         print("\nDocker Proxy:")
-        print("-------------------------\n")
+        print("-------------\n")
         print(textwrap.fill(
               "Docker proxy is needed if host OAM network is behind a proxy.",
               80))
@@ -2762,7 +2770,7 @@ class ConfigAssistant():
                             self.docker_http_proxy = user_input
                             break
                         else:
-                            print("Please input a valid url")
+                            print("Please enter a valid url")
                             continue
                     else:
                         self.docker_http_proxy = ""
@@ -2778,7 +2786,7 @@ class ConfigAssistant():
                             self.docker_https_proxy = user_input
                             break
                         else:
-                            print("Please input a valid url")
+                            print("Please enter a valid url")
                             continue
                     else:
                         self.docker_https_proxy = ""
@@ -2849,6 +2857,145 @@ class ConfigAssistant():
                 print("Invalid choice")
                 continue
 
+    def input_docker_registry_config(self):
+        """Allow user to input docker registry config."""
+
+        print("\nDocker Registry:")
+        print("----------------\n")
+        print("Configure docker registries to pull images from.\n"
+              "Default registries are:\n"
+              "k8s.gcr.io, gcr.io, quay.io, docker.io\n"
+              )
+        while True:
+            user_input = input(
+                "Use default docker registries [Y/n]: ")
+            if user_input.lower() == 'q':
+                raise UserQuit
+            elif user_input.lower() == 'n':
+                # unexpected newline displayed if textwrap.fill with
+                # '\n' included
+                print("\nEach registry can be specified as one of the"
+                      "following:\n"
+                      " - domain (e.g. example.domain)\n"
+                      " - domain with port (e.g. example.domain:5000)\n"
+                      " - IPv4 address (e.g. 1.2.3.4)\n"
+                      " - IPv4 address with port (e.g. 1.2.3.4:5000)\n"
+                      )
+                while True:
+                    user_input = input(
+                        "Use a unified registry replacing all "
+                        "default registries [y/n]: ")
+                    if user_input.lower() == 'q':
+                        raise UserQuit
+                    elif user_input.lower() == 'y':
+                        # Input a unified registry to avoid
+                        # inputing the same registry repeatly
+                        while True:
+                            user_input = input(
+                                "Enter a unified registry: ")
+                            if user_input.lower() == 'q':
+                                raise UserQuit
+                            if is_valid_domain_or_ip(user_input):
+                                self.docker_k8s_registry = user_input
+                                self.docker_gcr_registry = user_input
+                                self.docker_quay_registry = user_input
+                                self.docker_docker_registry = user_input
+                                self.docker_use_default_registry = False
+                                break
+                            else:
+                                print("Please enter a valid registry address")
+                                continue
+
+                        # Only if a unified registry set, it could be
+                        # an insecure registry
+                        while True:
+                            user_input = input(
+                                "Is '" + self.docker_k8s_registry +
+                                "' a secure registry (https) [Y/n]: ")
+                            if user_input.lower() == 'q':
+                                raise UserQuit
+                            elif user_input.lower() in ('y', ''):
+                                self.is_secure_registry = True
+                                break
+                            elif user_input.lower() == 'n':
+                                self.is_secure_registry = False
+                                break
+                            else:
+                                print("Invalid choice")
+                                continue
+                        break
+
+                    elif user_input.lower() == 'n':
+                        # Input alternative registries separately
+                        while True:
+                            user_input = input(
+                                "Alternative registry to k8s.gcr.io: ")
+                            if user_input.lower() == 'q':
+                                raise UserQuit
+                            if is_valid_domain_or_ip(user_input):
+                                self.docker_k8s_registry = user_input
+                                break
+                            else:
+                                print("Please enter a valid registry address")
+                                continue
+
+                        while True:
+                            user_input = input(
+                                "Alternative registry to gcr.io: ")
+                            if user_input.lower() == 'q':
+                                raise UserQuit
+                            if is_valid_domain_or_ip(user_input):
+                                self.docker_gcr_registry = user_input
+                                break
+                            else:
+                                print("Please enter a valid registry address")
+                                continue
+
+                        while True:
+                            user_input = input(
+                                "Alternative registry to quay.io: ")
+                            if user_input.lower() == 'q':
+                                raise UserQuit
+                            if is_valid_domain_or_ip(user_input):
+                                self.docker_quay_registry = user_input
+                                break
+                            else:
+                                print("Please enter a valid registry address")
+                                continue
+
+                        while True:
+                            user_input = input(
+                                "Alternative registry to docker.io: ")
+                            if user_input.lower() == 'q':
+                                raise UserQuit
+                            if is_valid_domain_or_ip(user_input):
+                                self.docker_docker_registry = user_input
+                                break
+                            else:
+                                print("Please enter a valid registry address")
+                                continue
+
+                        if (self.docker_k8s_registry or
+                                self.docker_gcr_registry or
+                                self.docker_quay_registry or
+                                self.docker_docker_registry):
+                            self.docker_use_default_registry = False
+                            break
+                        else:
+                            print("At least one registry is required")
+                            continue
+                    else:
+                        print("Invalid choice")
+                        continue
+                break
+
+            elif user_input.lower() in ('y', ''):
+                self.docker_use_default_registry = True
+                break
+            else:
+                print("Invalid choice")
+                continue
+
     def input_authentication_config(self):
         """Allow user to input authentication config and perform validation.
         """
@@ -2936,6 +3083,7 @@ class ConfigAssistant():
             self.input_dns_config()
             # Docker proxy is only used in kubernetes config
             self.input_docker_proxy_config()
+            self.input_docker_registry_config()
         self.input_authentication_config()
 
     def is_valid_management_multicast_subnet(self, ip_subnet):
@@ -3298,6 +3446,32 @@ class ConfigAssistant():
                         self.docker_no_proxy = config.get(
                             'cDOCKER_PROXY', 'DOCKER_NO_PROXY')
 
+                # Docker Registry Configuration
+                if config.has_section('cDOCKER_REGISTRY'):
+                    self.docker_use_default_registry = False
+                    if config.has_option('cDOCKER_REGISTRY',
+                                         'DOCKER_K8S_REGISTRY'):
+                        self.docker_k8s_registry = config.get(
+                            'cDOCKER_REGISTRY', 'DOCKER_K8S_REGISTRY')
+                    if config.has_option('cDOCKER_REGISTRY',
+                                         'DOCKER_GCR_REGISTRY'):
+                        self.docker_gcr_registry = config.get(
+                            'cDOCKER_REGISTRY', 'DOCKER_GCR_REGISTRY')
+                    if config.has_option('cDOCKER_REGISTRY',
+                                         'DOCKER_QUAY_REGISTRY'):
+                        self.docker_quay_registry = config.get(
+                            'cDOCKER_REGISTRY', 'DOCKER_QUAY_REGISTRY')
+                    if config.has_option('cDOCKER_REGISTRY',
+                                         'DOCKER_DOCKER_REGISTRY'):
+                        self.docker_docker_registry = config.get(
+                            'cDOCKER_REGISTRY', 'DOCKER_DOCKER_REGISTRY')
+                    if config.has_option('cDOCKER_REGISTRY',
+                                         'IS_SECURE_REGISTRY'):
+                        self.is_secure_registry = config.getboolean(
+                            'cDOCKER_REGISTRY', 'IS_SECURE_REGISTRY')
+                    else:
+                        self.is_secure_registry = True
+
             # SDN Network configuration
             if config.has_option('cSDN', 'ENABLE_SDN'):
                 raise ConfigFail("The option ENABLE_SDN is no longer "
@@ -3742,14 +3916,31 @@ class ConfigAssistant():
             if not dns_config:
                 print("External DNS servers not configured")
             if self.enable_docker_proxy:
-                print("\nDocker Proxy Configuraton")
+                print("\nDocker Proxy Configuration")
-                print("----------------------")
+                print("--------------------------")
                 if self.docker_http_proxy:
                     print("Docker HTTP proxy: " + self.docker_http_proxy)
                 if self.docker_https_proxy:
                     print("Docker HTTPS proxy: " + self.docker_https_proxy)
                 if self.docker_no_proxy:
                     print("Docker NO proxy: " + self.docker_no_proxy)
+            if not self.docker_use_default_registry:
+                print("\nDocker Registry Configuration")
+                print("-----------------------------")
+                if self.docker_k8s_registry:
+                    print("Alternative registry to k8s.gcr.io: " +
+                          self.docker_k8s_registry)
+                if self.docker_gcr_registry:
+                    print("Alternative registry to gcr.io: " +
+                          self.docker_gcr_registry)
+                if self.docker_quay_registry:
+                    print("Alternative registry to quay.io: " +
+                          self.docker_quay_registry)
+                if self.docker_docker_registry:
+                    print("Alternative registry to docker.io: " +
+                          self.docker_docker_registry)
+                print("Is registries secure: " +
+                      str(self.is_secure_registry))
 
         if self.region_config:
             print("\nRegion Configuration")
@@ -4064,6 +4255,30 @@ class ConfigAssistant():
                                 "DOCKER_NO_PROXY=" +
                                 str(self.docker_no_proxy) + "\n")
 
+                    # Docker registry configuration
+                    if not self.docker_use_default_registry:
+                        f.write("\n[cDOCKER_REGISTRY]")
+                        f.write("\n# Docker Registry Configuration\n")
+                        if self.docker_k8s_registry:
+                            f.write(
+                                "DOCKER_K8S_REGISTRY=" +
+                                str(self.docker_k8s_registry) + "\n")
+                        if self.docker_gcr_registry:
+                            f.write(
+                                "DOCKER_GCR_REGISTRY=" +
+                                str(self.docker_gcr_registry) + "\n")
+                        if self.docker_quay_registry:
+                            f.write(
+                                "DOCKER_QUAY_REGISTRY=" +
+                                str(self.docker_quay_registry) + "\n")
+                        if self.docker_docker_registry:
+                            f.write(
+                                "DOCKER_DOCKER_REGISTRY=" +
+                                str(self.docker_docker_registry) + "\n")
+                        f.write(
+                            "IS_SECURE_REGISTRY=" +
+                            str(self.is_secure_registry) + "\n")
+
                 # Network configuration
                 f.write("\n[cNETWORK]")
                 f.write("\n# Data Network Configuration\n")
@@ -5468,22 +5683,54 @@ class ConfigAssistant():
         client.sysinv.idns.update(dns_record.uuid, patch)
 
     def _populate_docker_config(self, client):
-        parameter = {}
+        if self.enable_docker_proxy:
-        if self.docker_http_proxy:
+            proxy_parameter = {}
-            parameter['http_proxy'] = self.docker_http_proxy
+            if self.docker_http_proxy:
-        if self.docker_https_proxy:
+                proxy_parameter['http_proxy'] = self.docker_http_proxy
-            parameter['https_proxy'] = self.docker_https_proxy
+            if self.docker_https_proxy:
-        if self.docker_no_proxy:
+                proxy_parameter['https_proxy'] = self.docker_https_proxy
-            parameter['no_proxy'] = self.docker_no_proxy
+            if self.docker_no_proxy:
+                proxy_parameter['no_proxy'] = self.docker_no_proxy
 
-        if parameter:
+            if proxy_parameter:
-            client.sysinv.service_parameter.create(
+                client.sysinv.service_parameter.create(
-                sysinv_constants.SERVICE_TYPE_DOCKER,
+                    sysinv_constants.SERVICE_TYPE_DOCKER,
-                sysinv_constants.SERVICE_PARAM_SECTION_DOCKER_PROXY,
+                    sysinv_constants.SERVICE_PARAM_SECTION_DOCKER_PROXY,
-                None,
+                    None,
-                None,
+                    None,
-                parameter
+                    proxy_parameter
-            )
+                )
+
+        if not self.docker_use_default_registry:
+            registry_parameter = {}
+            if self.docker_k8s_registry:
+                registry_parameter['k8s'] = \
+                    self.docker_k8s_registry
+
+            if self.docker_gcr_registry:
+                registry_parameter['gcr'] = \
+                    self.docker_gcr_registry
+
+            if self.docker_quay_registry:
+                registry_parameter['quay'] = \
+                    self.docker_quay_registry
+
+            if self.docker_docker_registry:
+                registry_parameter['docker'] = \
+                    self.docker_docker_registry
+
+            if not self.is_secure_registry:
+                registry_parameter['insecure_registry'] = "True"
+
+            if registry_parameter:
+                client.sysinv.service_parameter.create(
+                    sysinv_constants.SERVICE_TYPE_DOCKER,
+                    sysinv_constants.
+                    SERVICE_PARAM_SECTION_DOCKER_REGISTRY,
+                    None,
+                    None,
+                    registry_parameter
+                )
 
     def populate_initial_config(self):
         """Populate initial system inventory configuration"""
@@ -5494,8 +5741,7 @@ class ConfigAssistant():
                 self._populate_network_config(client)
                 if self.kubernetes:
                     self._populate_dns_config(client)
-                    if self.enable_docker_proxy:
+                    self._populate_docker_config(client)
-                        self._populate_docker_config(client)
                 controller = self._populate_controller_config(client)
                 # ceph_mon config requires controller host to be created
                 self._inventory_config_complete_wait(client, controller)

controllerconfig/controllerconfig/controllerconfig/tests/files/cgcs_config.kubernetes

@@ -79,6 +79,14 @@ DOCKER_HTTP_PROXY=http://proxy.com:123
 DOCKER_HTTPS_PROXY=https://proxy.com:123
 DOCKER_NO_PROXY=localhost,127.0.0.1,192.168.204.2
 
+[cDOCKER_REGISTRY]
+# Docker Registry Configuration
+DOCKER_K8S_REGISTRY=my.registry.com:5000
+DOCKER_GCR_REGISTRY=my.registry.com
+DOCKER_QUAY_REGISTRY=1.2.3.4:5000
+DOCKER_DOCKER_REGISTRY=[1:2:3:4:a:b:c:d]:5000
+IS_SECURE_REGISTRY=False
+
 [cNETWORK]
 # Data Network Configuration
 VSWITCH_TYPE=ovs-dpdk

controllerconfig/controllerconfig/controllerconfig/tests/files/system_config.kubernetes

@@ -55,6 +55,14 @@ DOCKER_HTTP_PROXY=http://proxy.com:123
 DOCKER_HTTPS_PROXY=https://proxy.com:123
 DOCKER_NO_PROXY=localhost,127.0.0.1,192.168.204.2
 
+[DOCKER_REGISTRY]
+# Docker Registry Configuration
+DOCKER_K8S_REGISTRY=my.registry.com:5000
+DOCKER_GCR_REGISTRY=my.registry.com
+DOCKER_QUAY_REGISTRY=1.2.3.4:5000
+DOCKER_DOCKER_REGISTRY=[1:2:3:4:a:b:c:d]:5000
+IS_SECURE_REGISTRY=False
+
 ;[PXEBOOT_NETWORK]
 ;PXEBOOT_CIDR=192.168.203.0/24
 

controllerconfig/controllerconfig/controllerconfig/tests/test_system_config.py

@@ -654,3 +654,10 @@ def test_kubernetes():
     cr.create_cgcs_config_file(None, system_config, None, None, None, 0,
                                validate_only=True)
     validate(system_config, DEFAULT_CONFIG, None, False)
+
+    # Test absence of optional docker registry configuration
+    system_config = cr.parse_system_config(systemfile)
+    system_config.remove_section('DOCKER_REGISTRY')
+    cr.create_cgcs_config_file(None, system_config, None, None, None, 0,
+                               validate_only=True)
+    validate(system_config, DEFAULT_CONFIG, None, False)

puppet-manifests/src/modules/platform/manifests/docker.pp

@@ -3,6 +3,11 @@ class platform::docker::params (
   $http_proxy   = undef,
   $https_proxy  = undef,
   $no_proxy     = undef,
+  $k8s_registry    = undef,
+  $gcr_registry    = undef,
+  $quay_registry   = undef,
+  $docker_registry = undef,
+  $insecure_registry    = undef,
 ) { }
 
 class platform::docker::config

puppet-manifests/src/modules/platform/manifests/dockerdistribution.pp

@@ -7,9 +7,18 @@ class platform::dockerdistribution::config
 
   if $enabled {
     include ::platform::network::mgmt::params
+    include ::platform::docker::params
 
     $docker_registry_ip = $::platform::network::mgmt::params::controller_address
 
+    # check insecure registries
+    if $::platform::docker::params::insecure_registry {
+      # insecure registry is true means unified registry was set
+      $insecure_registries = "\"${::platform::docker::params::k8s_registry}\", \"${docker_registry_ip}:9001\""
+    } else {
+      $insecure_registries = "\"${docker_registry_ip}:9001\""
+    }
+
     # currently docker registry is running insecure mode
     # when proper authentication is implemented, this would go away
     file { '/etc/docker':
@@ -52,9 +61,18 @@ class platform::dockerdistribution::compute
   $enabled = $::platform::kubernetes::params::enabled
   if $enabled {
     include ::platform::network::mgmt::params
+    include ::platform::docker::params
 
     $docker_registry_ip = $::platform::network::mgmt::params::controller_address
 
+    # check insecure registries
+    if $::platform::docker::params::insecure_registry {
+      # insecure registry is true means unified registry was set
+      $insecure_registries = "\"${::platform::docker::params::k8s_registry}\", \"${docker_registry_ip}:9001\""
+    } else {
+      $insecure_registries = "\"${docker_registry_ip}:9001\""
+    }
+
     # currently docker registry is running insecure mode
     # when proper authentication is implemented, this would go away
     file { '/etc/docker':

puppet-manifests/src/modules/platform/manifests/helm.pp

@@ -7,6 +7,7 @@ class platform::helm
   inherits ::platform::helm::repository::params {
 
   include ::platform::kubernetes::params
+  include ::platform::docker::params
 
   if $::platform::kubernetes::params::enabled {
     file {$source_helm_repo_dir:
@@ -27,17 +28,29 @@ class platform::helm
 
       if str2bool($::is_initial_config_primary) {
 
+        if $::platform::docker::params::gcr_registry {
+          $gcr_registry = $::platform::docker::params::gcr_registry
+        } else {
+          $gcr_registry = 'gcr.io'
+        }
+
+        if $::platform::docker::params::quay_registry {
+          $quay_registry = $::platform::docker::params::quay_registry
+        } else {
+          $quay_registry = 'quay.io'
+        }
+
         Class['::platform::kubernetes::master']
 
         # TODO(jrichard): Upversion tiller image to v2.11.1 once released.
         -> exec { 'load tiller docker image':
-          command   => 'docker image pull gcr.io/kubernetes-helm/tiller:v2.12.1',
+          command   => "docker image pull ${gcr_registry}/kubernetes-helm/tiller:v2.12.1",
           logoutput => true,
         }
 
         # TODO(tngo): If and when tiller image is upversioned, please ensure armada compatibility as part of the test
         -> exec { 'load armada docker image':
-          command   => 'docker image pull quay.io/airshipit/armada:f807c3a1ec727c883c772ffc618f084d960ed5c9',
+          command   => "docker image pull ${quay_registry}/airshipit/armada:f807c3a1ec727c883c772ffc618f084d960ed5c9",
           logoutput => true,
         }
 
@@ -54,7 +67,7 @@ class platform::helm
         # TODO(jrichard): Upversion tiller image to v2.11.1 once released.
         -> exec { 'initialize helm':
           environment => [ 'KUBECONFIG=/etc/kubernetes/admin.conf', 'HOME=/home/wrsroot' ],
-          command     => 'helm init --skip-refresh --service-account tiller --node-selectors "node-role.kubernetes.io/master"="" --tiller-image=gcr.io/kubernetes-helm/tiller:v2.12.1', # lint:ignore:140chars
+          command     => "helm init --skip-refresh --service-account tiller --node-selectors \"node-role.kubernetes.io/master\"=\"\" --tiller-image=${gcr_registry}/kubernetes-helm/tiller:v2.12.1", # lint:ignore:140chars
           logoutput   => true,
           user        => 'wrsroot',
           group       => 'wrs',

puppet-manifests/src/modules/platform/manifests/kubernetes.pp

@@ -13,9 +13,27 @@ class platform::kubernetes::params (
 ) { }
 
 class platform::kubernetes::kubeadm {
+  include ::platform::docker::params
+
   $iptables_file = "net.bridge.bridge-nf-call-ip6tables = 1
     net.bridge.bridge-nf-call-iptables = 1"
 
+  if $::platform::docker::params::k8s_registry {
+    $k8s_registry = $::platform::docker::params::k8s_registry
+  } else {
+    $k8s_registry = undef
+  }
+
+  # kubelet use --pod-infra-container-image to indentify the specified image
+  # TODO: this is not needed after kubernetes upgraded to 1.13
+  #       because the imageRepository setting will be used
+  if $k8s_registry {
+    file { '/etc/sysconfig/kubelet':
+      ensure  => file,
+      content => template('platform/kubelet.conf.erb'),
+    }
+  }
+
   # Update iptables config. This is required based on:
   # https://kubernetes.io/docs/tasks/tools/install-kubeadm
   # This probably belongs somewhere else - initscripts package?
@@ -52,6 +70,21 @@ class platform::kubernetes::master::init
   inherits ::platform::kubernetes::params {
 
   include ::platform::params
+  include ::platform::docker::params
+
+  # This is used for imageRepository in template kubeadm.yaml.erb
+  if $::platform::docker::params::k8s_registry {
+    $k8s_registry = $::platform::docker::params::k8s_registry
+  } else {
+    $k8s_registry = undef
+  }
+
+  # This is used for calico image in template calico.yaml.erb
+  if $::platform::docker::params::quay_registry {
+    $quay_registry = $::platform::docker::params::quay_registry
+  } else {
+    $quay_registry = 'quay.io'
+  }
 
   if str2bool($::is_initial_config_primary) {
     # For initial controller install, configure kubernetes from scratch.
@@ -347,10 +380,11 @@ class platform::kubernetes::worker
 }
 
 # TODO: remove port 9001 once we have a public docker image registry using standard ports.
+# add 5000 as the default port for private registry
 class platform::kubernetes::firewall::params (
   $transport = 'tcp',
   $table = 'nat',
-  $dports = [80, 443, 9001],
+  $dports = [80, 443, 9001, 5000],
   $chain = 'POSTROUTING',
   $jump = 'SNAT',
 ) {}

puppet-manifests/src/modules/platform/templates/calico.yaml.erb

@@ -107,7 +107,7 @@ spec:
       # as a host-networked pod.
       serviceAccountName: calico-node
       containers:
-      - image: quay.io/calico/typha:v3.1.4
+      - image: <%= @quay_registry %>/calico/typha:v3.1.4
         name: calico-typha
         ports:
         - containerPort: 5473
@@ -198,7 +198,7 @@ spec:
         # container programs network policy and routes on each
         # host.
         - name: calico-node
-          image: quay.io/calico/node:v3.1.4
+          image: <%= @quay_registry %>/calico/node:v3.1.4
           env:
             # Use Kubernetes API as the backing datastore.
             - name: DATASTORE_TYPE
@@ -283,7 +283,7 @@ spec:
         # This container installs the Calico CNI binaries
         # and CNI network config file on each node.
         - name: install-cni
-          image: quay.io/calico/cni:v3.1.4
+          image: <%= @quay_registry %>/calico/cni:v3.1.4
           command: ["/install-cni.sh"]
           env:
             # Name of the CNI config file to create.

puppet-manifests/src/modules/platform/templates/insecuredockerregistry.conf.erb

@@ -1,3 +1,3 @@
 {
-    "insecure-registries" : [ "<%= @docker_registry_ip %>:9001" ]
+    "insecure-registries" : [ <%= @insecure_registries %> ]
 }

puppet-manifests/src/modules/platform/templates/kubeadm.yaml.erb

@@ -21,6 +21,9 @@ controllerManagerExtraArgs:
   node-monitor-period: "2s"
   node-monitor-grace-period: "20s"
   pod-eviction-timeout: "30s"
+<%- if @k8s_registry -%>
+imageRepository: "<%= @k8s_registry %>"
+<%- end -%>
 ---
 kind: KubeletConfiguration
 apiVersion: kubelet.config.k8s.io/v1beta1

puppet-manifests/src/modules/platform/templates/kubelet.conf.erb

@@ -0,0 +1,2 @@
+# Overrides config file for kubelet
+KUBELET_EXTRA_ARGS=--pod-infra-container-image=<%= @k8s_registry %>/pause:3.1

puppet-modules-wrs/puppet-sysinv/src/sysinv/manifests/init.pp

@@ -77,6 +77,7 @@ class sysinv (
 
   include sysinv::params
   include ::platform::kubernetes::params
+  include ::platform::docker::params
 
   Package['sysinv'] -> Sysinv_config<||>
   Package['sysinv'] -> Sysinv_api_paste_ini<||>
@@ -217,7 +218,13 @@ class sysinv (
   }
 
   if $::platform::kubernetes::params::enabled == true {
-    $armada_img_tag = 'quay.io/airshipit/armada:f807c3a1ec727c883c772ffc618f084d960ed5c9'
+    if $::platform::docker::params::quay_registry {
+      $quay_registry = $::platform::docker::params::quay_registry
+    } else {
+      $quay_registry = 'quay.io'
+    }
+
+    $armada_img_tag = "${quay_registry}/airshipit/armada:f807c3a1ec727c883c772ffc618f084d960ed5c9"
     sysinv_config {
       'DEFAULT/armada_image_tag':    value =>  $armada_img_tag;
     }

sysinv/sysinv/sysinv/sysinv/common/constants.py

@@ -1107,6 +1107,12 @@ SERVICE_PARAM_SECTION_DOCKER_PROXY = 'proxy'
 SERVICE_PARAM_NAME_DOCKER_HTTP_PROXY = 'http_proxy'
 SERVICE_PARAM_NAME_DOCKER_HTTPS_PROXY = 'https_proxy'
 SERVICE_PARAM_NAME_DOCKER_NO_PROXY = 'no_proxy'
+SERVICE_PARAM_SECTION_DOCKER_REGISTRY = 'registry'
+SERVICE_PARAM_NAME_DOCKER_K8S_REGISTRY = 'k8s'
+SERVICE_PARAM_NAME_DOCKER_GCR_REGISTRY = 'gcr'
+SERVICE_PARAM_NAME_DOCKER_QUAY_REGISTRY = 'quay'
+SERVICE_PARAM_NAME_DOCKER_DOCKER_REGISTRY = 'docker'
+SERVICE_PARAM_NAME_DOCKER_INSECURE_REGISTRY = 'insecure_registry'
 
 # default filesystem size to 25 MB
 SERVICE_PARAM_SWIFT_FS_SIZE_MB_DEFAULT = 25

sysinv/sysinv/sysinv/sysinv/common/service_parameter.py

@@ -660,11 +660,24 @@ def _validate_docker_no_proxy_address(name, value):
     values = value.split(',')
     for item in values:
         # will extend to more cases if CIDR notation is supported
-        if not cutils.is_valid_domain(item):
+        if not cutils.is_valid_domain_or_ip(item):
-            if not cutils.is_valid_ip(item):
+            raise wsme.exc.ClientSideError(_(
-                raise wsme.exc.ClientSideError(_(
+                "Parameter '%s' includes an invalid address '%s'." %
-                    "Parameter '%s' includes an invalid address '%s'." %
+                (name, item)))
-                    (name, item)))
+
+
+def _validate_docker_registry_address(name, value):
+    """Check if registry address is valid"""
+    if not cutils.is_valid_domain_or_ip(value):
+        raise wsme.exc.ClientSideError(_(
+            "Parameter '%s' must be a valid address." % name))
+
+
+def _validate_docker_insecure_registry_bool(name, value):
+    """Check if insecure registry is a valid bool"""
+    if not cutils.is_valid_boolstr(value):
+        raise wsme.exc.ClientSideError(_(
+            "Parameter '%s' must be a valid bool string." % name))
 
 
 # LDAP Identity Service Parameters (mandatory)
@@ -1516,6 +1529,35 @@ DOCKER_PROXY_PARAMETER_RESOURCE = {
         'platform::docker::params::no_proxy',
 }
 
+DOCKER_REGISTRY_PARAMETER_OPTIONAL = [
+    constants.SERVICE_PARAM_NAME_DOCKER_K8S_REGISTRY,
+    constants.SERVICE_PARAM_NAME_DOCKER_GCR_REGISTRY,
+    constants.SERVICE_PARAM_NAME_DOCKER_QUAY_REGISTRY,
+    constants.SERVICE_PARAM_NAME_DOCKER_DOCKER_REGISTRY,
+    constants.SERVICE_PARAM_NAME_DOCKER_INSECURE_REGISTRY,
+]
+
+DOCKER_REGISTRY_PARAMETER_VALIDATOR = {
+    constants.SERVICE_PARAM_NAME_DOCKER_K8S_REGISTRY: _validate_docker_registry_address,
+    constants.SERVICE_PARAM_NAME_DOCKER_GCR_REGISTRY: _validate_docker_registry_address,
+    constants.SERVICE_PARAM_NAME_DOCKER_QUAY_REGISTRY: _validate_docker_registry_address,
+    constants.SERVICE_PARAM_NAME_DOCKER_DOCKER_REGISTRY: _validate_docker_registry_address,
+    constants.SERVICE_PARAM_NAME_DOCKER_INSECURE_REGISTRY: _validate_docker_insecure_registry_bool,
+}
+
+DOCKER_REGISTRY_PARAMETER_RESOURCE = {
+    constants.SERVICE_PARAM_NAME_DOCKER_K8S_REGISTRY:
+        'platform::docker::params::k8s_registry',
+    constants.SERVICE_PARAM_NAME_DOCKER_GCR_REGISTRY:
+        'platform::docker::params::gcr_registry',
+    constants.SERVICE_PARAM_NAME_DOCKER_QUAY_REGISTRY:
+        'platform::docker::params::quay_registry',
+    constants.SERVICE_PARAM_NAME_DOCKER_DOCKER_REGISTRY:
+        'platform::docker::params::docker_registry',
+    constants.SERVICE_PARAM_NAME_DOCKER_INSECURE_REGISTRY:
+        'platform::docker::params::insecure_registry',
+}
+
 HTTPD_PORT_PARAMETER_OPTIONAL = [
     constants.SERVICE_PARAM_HTTP_PORT_HTTP,
     constants.SERVICE_PARAM_HTTP_PORT_HTTPS,
@@ -1715,6 +1757,11 @@ SERVICE_PARAMETER_SCHEMA = {
             SERVICE_PARAM_VALIDATOR: DOCKER_PROXY_PARAMETER_VALIDATOR,
             SERVICE_PARAM_RESOURCE: DOCKER_PROXY_PARAMETER_RESOURCE,
         },
+        constants.SERVICE_PARAM_SECTION_DOCKER_REGISTRY: {
+            SERVICE_PARAM_OPTIONAL: DOCKER_REGISTRY_PARAMETER_OPTIONAL,
+            SERVICE_PARAM_VALIDATOR: DOCKER_REGISTRY_PARAMETER_VALIDATOR,
+            SERVICE_PARAM_RESOURCE: DOCKER_REGISTRY_PARAMETER_RESOURCE,
+        },
     },
     constants.SERVICE_TYPE_HTTP: {
         constants.SERVICE_PARAM_SECTION_HTTP_CONFIG: {

sysinv/sysinv/sysinv/sysinv/common/utils.py

@@ -1832,6 +1832,33 @@ def is_valid_domain(url_str):
         return False
 
 
+def is_valid_domain_or_ip(url_str):
+    if url_str:
+        if is_valid_domain(url_str):
+            return True
+        ip_with_port = url_str.split(':')
+        if len(ip_with_port) <= 2:
+            # check ipv4 or ipv4 with port
+            return is_valid_ipv4(ip_with_port[0])
+        else:
+            # check ipv6 with port
+            if '[' in url_str:
+                try:
+                    bkt_idx = url_str.index(']')
+                    if bkt_idx + 1 == len(url_str):
+                        # brackets without port
+                        return False
+                    else:
+                        return is_valid_ipv6(url_str[1:bkt_idx])
+                except Exception:
+                    return False
+            else:
+                # check ipv6 without port
+                return is_valid_ipv6(url_str)
+    else:
+        return False
+
+
 def verify_checksum(path):
     """ Find and validate the checksum file in a given directory. """
     rc = True